Add files via upload

Author: ashishoist91

Employee.java

@@ -0,0 +1,52 @@
+import java.io.Serializable;
+
+public class Employee implements Serializable {
+	
+	/**
+	 * 
+	 */
+	private static final long serialVersionUID = 1L;
+	private String employeeName;
+	private String department;
+	private String city;
+	private Integer salary;
+	
+	
+	
+	
+	public Employee(String employeeName, String department, String city, Integer salary) {
+		super();
+		this.employeeName = employeeName;
+		this.department = department;
+		this.city = city;
+		this.salary = salary;
+	}
+	
+	public String getEmployeeName() {
+		return employeeName;
+	}
+	public void setEmployeeName(String employeeName) {
+		this.employeeName = employeeName;
+	}
+	public String getDepartment() {
+		return department;
+	}
+	public void setDepartment(String department) {
+		this.department = department;
+	}
+	public String getCity() {
+		return city;
+	}
+	public void setCity(String city) {
+		this.city = city;
+	}
+	public Integer getSalary() {
+		return salary;
+	}
+	public void setSalary(Integer salary) {
+		this.salary = salary;
+	}
+	
+	
+
+}

SparkSQL.java

@@ -0,0 +1,31 @@
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.SQLContext;
+
+public class SparkSQL {
+
+	private static Integer getNumericSal(String salary) {
+
+		return Integer.parseInt(salary.replace("$", "").replace(",", "").trim());
+		
+	}
+	
+	public static void main(String[] args) {
+		JavaSparkContext javaSparkContext = SparkUtility.getJavaSparkContext(SparkConstant.MASTER_LOCAL, SparkConstant.APP_NAME+"SparkSQL");
+		@SuppressWarnings("unused")
+		SQLContext sqlContext = new org.apache.spark.sql.SQLContext(javaSparkContext);
+		JavaRDD<Employee> employee = javaSparkContext.textFile("file:///home/hduser/Files/EmpSal").map(line ->{
+			String arr[] = line.split("\t");
+			return new Employee(arr[0],arr[1],arr[3], getNumericSal(arr[5]));
+		});
+		DataFrame employeeDF = sqlContext.createDataFrame(employee, Employee.class);
+		employeeDF.show();
+		employeeDF.printSchema();
+		employeeDF.registerTempTable("employee");
+		DataFrame empResult = sqlContext.sql("SELECT * FROM employee WHERE salary>40728");
+		//empResult.collectAsList().forEach(System.out.prin);
+		empResult.collectAsList().forEach(System.out::println);
+	}
+
+}

SparkSQLHive.java

@@ -0,0 +1,9 @@
+
+public class SparkSQLHive {
+
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
+
+	}
+
+}

SparkSQLJSON.java

@@ -0,0 +1,29 @@
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.SQLContext;
+
+public class SparkSQLJSON {
+
+	public static void main(String[] args) {
+		JavaSparkContext javaSparkContext = SparkUtility.getJavaSparkContext(SparkConstant.MASTER_LOCAL, SparkConstant.APP_NAME+"SparkSQL");
+		SQLContext sqlContext = new org.apache.spark.sql.SQLContext(javaSparkContext);
+		DataFrame employeeDF = sqlContext.read().json("file:///home/hduser/Files/Employee.json");
+		
+		employeeDF.show();
+		employeeDF.printSchema();
+		employeeDF.registerTempTable("employee");
+		DataFrame empResult = sqlContext.sql("SELECT name, address.city FROM employee WHERE address.state='California'");
+		//empResult.collectAsList().forEach(System.out.prin);
+		empResult.collectAsList().forEach(System.out::println);
+		//employeeDF.saveAsParquetFile("");
+		
+		employeeDF
+	      .write()
+	      .format("parquet")
+	      .save("file:///home/hduser/Files/Employee.parquet");
+		
+		//employeeDF.write().save("file:///home/hduser/Files/Employeeparquet");
+	}
+
+}

SparkSQLParquet.java

@@ -0,0 +1,27 @@
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.SQLContext;
+
+public class SparkSQLParquet {
+
+	public static void main(String[] args) {
+		JavaSparkContext javaSparkContext = SparkUtility.getJavaSparkContext(SparkConstant.MASTER_LOCAL, SparkConstant.APP_NAME+"SparkSQL");
+		SQLContext sqlContext = new org.apache.spark.sql.SQLContext(javaSparkContext);
+		DataFrame employeeDF = sqlContext.parquetFile("file:///home/hduser/Files/Employee.parquet/Employee.parquet");
+		
+		employeeDF.show();
+		employeeDF.printSchema();
+		employeeDF.registerTempTable("employee");
+		DataFrame empResult = sqlContext.sql("SELECT name, address.city FROM employee WHERE address.state='California'");
+		//empResult.collectAsList().forEach(System.out.prin);
+		empResult.collectAsList().forEach(System.out::println);
+		//employeeDF.saveAsParquetFile("");
+		
+//		employeeDF
+//	      .write()
+//	      .format("parquet")
+//	      .save("file:///home/hduser/Files/Employee.parquet");
+
+	}
+
+}