change files

Author: bolei

.sconsign.dblite

@@ -39,18 +39,18 @@ if scala_home == None:
 	sys.exit()
 
 # starting to build spark project
-spark_process = subprocess.Popen('sbt/sbt package', shell = True, executable = "/bin/bash")
+spark_process = subprocess.Popen('sbt/sbt assembly', shell = True, executable = "/bin/bash")
 spark_process.wait()
 
 if spark_process.returncode != 0:
-	print "Buil spark error ! \n"
+	print "Build spark error ! \n"
 	sys.exit()
 
 #decompress spark jar 
-if os.path.exists(home + '/.class'):
-    command = 'cd ' + home + '/.class && jar -xf ' + current + '/core/target/spark*.jar && cd ' + current
+if os.path.exists(home + '/.classes'):
+    command = 'cd ' + home + '/.classes && jar -xf ' + current + '/core/target/spark*.jar && cd ' + current
 else:
-    command = 'mkdir ' + home + '/.class && cd ' + home + '/.class && jar -xf ' + current + '/core/target/spark*.jar && cd ' + current
+    command = 'mkdir ' + home + '/.classes && cd ' + home + '/.classes && jar -xf ' + current + '/core/target/spark*.jar && cd ' + current
 
 spark_process = subprocess.Popen(command , shell = True, executable = "/bin/bash")
 spark_process.wait()
@@ -60,55 +60,45 @@ os.environ['SPARK_CLASS_PATH'] = home + '/.class'
 
 env = env.Append(CPPPATH = [java_home + '/include', 
 							java_home + '/include/linux',
-							java_home + '/jre/lib/amd64/server'])
+							java_home + '/jre/lib/amd64/server',
+							current + '/c++/'],
+				LIBS = ['jvm', 'pthread'],
+				LIBPATH = [java_home + '/jre/lib/amd64/server'])
 
-sources = Glob('*.c*')
-spark_jni_so = env.SharedLibrary('#c++/spark/jni/spark_jni',
-				sources,
-				LIBS = 'jvm',
-				srcdir = '#c++/spark/jni')
+print 'starting to compile c++ API code ....'
 
-spark_api_so = env.SharedLibrary('#c++/spark/spark_api',
-				sources,
-				LIBS = 'jvm',
-				srcdir = '#c++/spark/')
+jni_sources = Glob('#c++/spark/jni/*.c*')
 
-header_files = Glob('#c++/spark/*.h*')
+spark_jni_obj = Object(jni_sources, srcdir = '#c++/spark/jni')
 
-env.Alias('install-lib', Install(os.path.join(prefix, "/lib"), spark_jni_so, spark_api_so))
-env.Alias('install-include', Install(os.path.join(prefix, "/include"), header_files))
-env.Alias('install', ['install-include', 'install-lib'])
+spark_jni_so = SharedLibrary('#c++/spark/jni/spark_jni',spark_jni_obj)
 
-from SCons.Script.SConscript import SConsEnvironment
-SConsEnvironment.Chmod = SCons.ActionFactory(os.chmod, lambda dest, mode: 'Chmod("%s", "0%o")' % (dest, mode))
+api_sources = Glob('#c++/spark/*.c*')
 
-def InstallPerm(env, dest, files, perm):
-	obj = env.Install(dest, files)
-	for i in obj:
-		env.AddPostAction(i, env.Chmod(str(i), perm))
-	return dest
+spark_api_obj = Object(api_sources, srcdir = '#c++/spark')
 
-SConsEnvironment.InstallPerm = InstallPerm
-SConsEnvironment.InstallProgram = lambda env, dest, files: InstallPerm(env, dest, files, 0755)
-SConsEnvironment.InstallHeader = lambda env, dest, files: InstallPerm(env, dest, files, 0644)
+spark_api_so = SharedLibrary('#c++/spark/spark_api',spark_api_obj)
 
 header_files = Glob('#c++/spark/*.h*')
-env.InstallHeader(include_dir, header_files)
 
-try:
-	umask = os.umask(022)
-	print 'setting umask to 022 (was )%o)' % umask
-except:
-	pass
+#Alias('install-lib', Install(os.path.join(prefix, "/lib"), spark_jni_so, spark_api_so))
+#Alias('install-include', Install(os.path.join(prefix, "/include"), header_files))
+#Alias('install', ['install-include', 'install-lib'])
+
+command = 'cp c++/spark/*.h* /usr/local/include/spark'
+subprocess.Popen(command, shell = True, executable = "/bin/bash")
+command = 'cp c++/spark/jni/*.h* /usr/local/include/spark/jni'
+subprocess.Popen(command, shell = True, executable = "/bin/bash")
 
+print 'starting to compile examples code ....'
 examples_dir = current + '/c++/examples'
 
-env.Program('#c++/examples/wordcount',
-		'#c++/examples/wordcount.cpp',
+Program('#c++/examples/wordcount',
+		'wordcount.cpp',
 		LIBS = ['spark_api', 'jvm'],
 		srcdir = '#c++/examples/')
 
-env.Program('#c++/examples/kmeans',
-		'#c++/examples/kmeans.cpp',
+Program('#c++/examples/kmeans',
+		'kmeans.cpp',
 		LIBS = ['spark_api', 'jvm'],
 		srcdir = '#c++/examples/')

SConstruct

@@ -39,7 +39,7 @@ if scala_home == None:
 	sys.exit()
 
 # starting to build spark project
-spark_process = subprocess.Popen('sbt/sbt package', shell = True, executable = "/bin/bash")
+spark_process = subprocess.Popen('sbt/sbt assembly', shell = True, executable = "/bin/bash")
 spark_process.wait()
 
 if spark_process.returncode != 0:
@@ -60,9 +60,14 @@ os.environ['SPARK_CLASS_PATH'] = home + '/.class'
 
 env = env.Append(CPPPATH = [java_home + '/include', 
 							java_home + '/include/linux',
-							java_home + '/jre/lib/amd64/server'])
+							java_home + '/jre/lib/amd64/server'],
+				LIBS = ['jvm', 'pthread'],
+				LIBPATH = [java_home + '/jre/lib/amd64/server'])
 
 sources = Golb('*.c*')
+
+spark_jni_obj = env.Object(sources)
+
 spark_jni_so = env.SharedLibrary('#c++/spark/jni/spark_jni',
 				sources,
 				LIBS = 'jvm',

c++/SConstruct

@@ -1,15 +1,14 @@
 CXX = g++
 CC = gcc
 
-INCLUDES = -I/home/software/java/jdk1.6.0_37/include \
-		-I/home/software/java/jdk1.6.0_37/include/linux \
-		-I/home/software/java/jdk1.6.0_37/jre/lib/amd64/server/ \
-		-I/home/bolei/workspace/spark-0.6.0/c++/ \
-		-I/home/bolei/workspace/spark-0.6.0/c++/spark \
-		-I/usr/local/lib
+INCLUDES = -I$(JAVA_HOME)/include \
+		-I$(JAVA_HOME)/include/linux \
+		-I$(JAVA_HOME)/jre/lib/amd64/server/ \
+		-I$(CURDIR) \
+		-I/usr/local/include
 
 CLIBS =	-lsparkAPI \
-		-L/home/software/java/jdk1.6.0_37/jre/lib/amd64/server/ -ljvm
+		-L$(JAVA_HOME)/jre/lib/amd64/server/ -ljvm
 
 EXEC_OBJ = kmeans.o
 

c++/examples/client

@@ -1,14 +1,14 @@
 CXX = g++
 CC = gcc
 
-INCLUDES = -I/home/software/java/jdk1.6.0_37/include \
-		-I/home/software/java/jdk1.6.0_37/include/linux \
-		-I/home/software/java/jdk1.6.0_37/jre/lib/amd64/server/ \
-		-I/home/bolei/workspace/spark-0.6.0/c++ \
-		-I/home/bolei/workspace/spark-0.6.0/c++/spark 
+INCLUDES = -I$(JAVA_HOME)/include \
+		-I$(JAVA_HOME)/include/linux \
+		-I$(JAVA_HOME)/jre/lib/amd64/server/ \
+		-I$(CURDIR) \
+		-I/usr/local/include
 
 CLIBS =	-L/usr/local/lib -lsparkAPI \
-		-L/home/software/java/jdk1.6.0_37/jre/lib/amd64/server/ -ljvm 
+		-L$(JAVA_HOME)/jre/lib/amd64/server/ -ljvm 
 
 EXEC_OBJ = main.o 
 

c++/examples/kmeans

@@ -90,6 +90,21 @@ class Class{
 	virtual ~Class();
 };
 
+
+template<class T>
+class Partitioner : public Object {
+public:
+	Partitioner(int num) : numPartitions(num) {}
+	virtual ~Partitioner(){}
+
+	virtual int getPartitions(T t) = 0;
+	virtual bool equals(T t) = 0;
+
+public:
+	int numPartitions;
+};
+
+
 template<class T, class R>
 class MapContext : public Object {
 	public:
@@ -147,13 +162,6 @@ class Attachment {
 		T _attach;
 };
 
-
-class Partitioner : public Object {
-public:
-	Partitioner(){}
-	virtual ~Partitioner(){}
-};
-
 #endif // Class.h
 
 

c++/examples/kmeans.mk

@@ -1,11 +1,21 @@
 CXX = g++
 CC = gcc
-INCLUDES = -I/home/software/java/jdk1.6.0_37/include \
-		-I/home/software/java/jdk1.6.0_37/include/linux \
-		-I/home/software/java/jdk1.6.0_37/jre/lib/amd64/server/ \
-		-I/home/bolei/workspace/spark-0.6.0/c++/
 
-CLIBS =	-L/home/software/java/jdk1.6.0_37/jre/lib/amd64/server/ -ljvm -lpthread
+HEADER := $(shell cd ../ && pwd)
+
+INCLUDES = -I$(JAVA_HOME)/include \
+		-I$(JAVA_HOME)/include/linux \
+		-I$(JAVA_HOME)/jre/lib/amd64/server/ \
+		-I${HEADER}
+
+#.PHONY: print_vars
+
+#print_vars:
+#	@echo $(JAVA_HOME)
+#	@echo ${CURDIR}
+#	@echo $(HEADER)
+
+CLIBS =	-L$(JAVA_HOME)/jre/lib/amd64/server/ -ljvm -lpthread
 
 OBJS = convert.o \
 	   crdd.o \