python product example 1 version

.gitignore

@@ -53,3 +53,4 @@ thirdparty/*
 !thirdparty/install.sh
 !include/singa
 doc/
+/Debug/

Makefile.am

@@ -38,8 +38,7 @@ CUDNN_SRCS := src/neuralnet/loss_layer/cudnn_softmaxloss.cc \
 			  src/neuralnet/neuron_layer/cudnn_lrn.cc \
 			  src/neuralnet/neuron_layer/cudnn_convolution.cc
 
-PY_SRCS := tool/python/singa/driver_wrap.cxx \
-		   src/driver.cc
+PY_SRCS := tool/python/singa/driver_wrap.cxx 
 
 HDFS_SRCS := src/io/hdfsfile.cc \
 			 src/io/hdfsfile_store.cc

examples/cifar10/job.conf

@@ -1,10 +1,10 @@
 name: "cifar10-convnet"
-train_steps: 1000
-test_steps: 100
-test_freq: 200
+train_steps: 50000
+test_steps: 1000
+test_freq: 2000
 #validate_steps: 100
 #validate_freq: 300
-disp_freq: 50
+disp_freq: 1000
 #checkpoint_path: "examples/cifar10/checkpoint/step1000-worker0"
 train_one_batch {
   alg: kBP
@@ -31,8 +31,8 @@ neuralnet {
     type: kRecordInput
     store_conf {
       backend: "kvfile"
-      path: "examples/cifar10/train_data.bin"
-      mean_file: "examples/cifar10/image_mean.bin"
+      path: "/home/aaron/Projects/incubator-singa/examples/cifar10/train_data.bin"
+      mean_file: "/home/aaron/Projects/incubator-singa/examples/cifar10/image_mean.bin"
       batchsize: 100
       #random_skip: 5000
       shape: 3
@@ -61,8 +61,8 @@ neuralnet {
     type: kRecordInput
     store_conf {
       backend: "kvfile"
-      path: "examples/cifar10/test_data.bin"
-      mean_file: "examples/cifar10/image_mean.bin"
+      path: "/home/aaron/Projects/incubator-singa/examples/cifar10/test_data.bin"
+      mean_file: "/home/aaron/Projects/incubator-singa/examples/cifar10/image_mean.bin"
       batchsize: 100
       shape: 3
       shape: 32
@@ -275,5 +275,5 @@ cluster {
   nserver_groups: 1
   nworkers_per_group: 1
   nworkers_per_procs: 1
-  workspace: "examples/cifar10"
+  workspace: "/home/aaron/Projects/incubator-singa/examples/cifar10"
 }

examples/cifar10_mesos/cifarBin2img.py

@@ -0,0 +1,67 @@
+
+import sys, os
+from numpy.core.test_rational import numerator
+
+SINGA_ROOT=os.path.join(os.path.dirname(__file__),'../','../')
+sys.path.append(os.path.join(SINGA_ROOT,'tool','python'))
+from singa.model import * 
+from singa.utils import imgtool
+from PIL import Image
+import cPickle
+
+def unpickle(file):
+    fo = open(file, 'rb')
+    dict = cPickle.load(fo)
+    fo.close()
+    return dict
+
+def test():
+    '''
+        test imgtool toBin and toImg
+    '''
+    im = Image.open("dog.jpg").convert("RGB")
+
+    byteArray=imgtool.toBin(im,(32,32))
+    im2 = imgtool.toImg(byteArray,(32,32))
+
+    im2.save("dog2.jpg", "JPEG")
+
+
+def getLabelMap(path):
+    d = unpickle(path)
+    label_map=dict()
+    for index,line in numerator(d["label_names"]):
+        print index,line
+        label_map[index]=line
+    return label_map
+
+def generateImage(input_path,output_path,label_map,random):
+    dict=unpickle(input_path)
+    data=dict["data"]
+    labels=dict["labels"]
+    for index,d in numerator(data):
+        im = imgtool.toImg(data[index],(32,32))
+        temp_folder=os.path.join(output_path,label_map[labels[index]])
+        try:
+            os.stat(temp_folder)
+        except:
+            os.makedirs(temp_folder)
+        im.save(os.path.join(temp_folder,random+"_"+str(index)+".jpg"),"JPEG") 
+    #print labels
+
+def main():
+    label_map=getLabelMap("data/batches.meta")
+    generateImage("data/data_batch_1", "data/output",label_map,"1")
+    generateImage("data/data_batch_2", "data/output",label_map,"2")
+    generateImage("data/data_batch_3", "data/output",label_map,"3") 
+    generateImage("data/data_batch_4", "data/output",label_map,"4")
+    generateImage("data/data_batch_5", "data/output",label_map,"5")
+    generateImage("data/test_batch", "data/output",label_map,"6")
+
+if __name__=='__main__':
+    main()
+
+
+
+
+

examples/cifar10_mesos/entry.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+cd /workspace
+wget $1
+tar zxf *.tar.gz
+cp /workspace/model.py /usr/src/incubator-singa/tool/python/examples/user1-cifar10/
+if [ $2 -eq '2' ]; then
+  cd /usr/src/incubator-singa/examples/cifar10_mesos/
+  python main.py
+else
+  cd /usr/src/incubator-singa/
+  python tool/python/examples/user1-cifar10/main.py $3 $4 $5 $6 $7
+fi

examples/cifar10_mesos/executor.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import threading
+import time
+import glob
+from multiprocessing import Process
+
+MESOS_ROOT="/home/aaron/Softs/mesos-0.27.0/build"
+for egg in glob.glob(os.path.join(MESOS_ROOT,'src','python','dist','*.egg')):
+    sys.path.append(os.path.abspath(egg))
+
+
+import mesos.interface
+from mesos.interface import mesos_pb2
+import mesos.native
+
+import main
+
+class MyExecutor(mesos.interface.Executor):
+    def launchTask(self, driver, task):
+        # Create a thread to run the task. Tasks should always be run in new
+        # threads or processes, rather than inside launchTask itself.
+
+	def runTask(driver,task):
+            # We are in the child.
+            # This is where one would perform the requested task.
+
+            try:
+		print "test"
+                sys.argv.append("-singa_conf")
+		sys.argv.append("/home/aaron/Projects/incubator-singa/conf/singa.conf")
+                model = main.buildModel(1)
+                main.product(model)
+            except Exception as e:
+		print str(e)
+                print "Sending status failed..."
+                update = mesos_pb2.TaskStatus()
+                update.task_id.value = task.task_id.value
+                update.state = mesos_pb2.TASK_FAILED
+                update.data = 'data with a \0 byte' 
+                driver.sendStatusUpdate(update)
+                print "Sent status failed"
+                sys.exit(1) 
+            print "Sending status finished..."
+            update = mesos_pb2.TaskStatus()
+            update.task_id.value = task.task_id.value
+            update.state = mesos_pb2.TASK_FINISHED
+            update.data = 'data with a \0 byte'
+            driver.sendStatusUpdate(update)
+            print "Sent status finished"
+        print "Running task %s" % task.task_id.value
+        update = mesos_pb2.TaskStatus()
+        update.task_id.value = task.task_id.value
+        update.state = mesos_pb2.TASK_RUNNING
+        update.data = 'data with a \0 byte'
+        driver.sendStatusUpdate(update)
+        time.sleep(20)
+        p = Process(target=runTask, args=(driver,task))
+        p.start()
+        p.join()
+        '''
+        pid = os.fork()
+        if pid == 0:
+            # We are in the child.
+            # This is where one would perform the requested task.
+
+            try:
+                model = main.buildModel(1)
+		print "test"
+                sys.argv.append("-singa_conf")
+		sys.argv.append("/home/aaron/Projects/incubator-singa/conf/singa.conf")
+                main.product(model)
+            except Exception as e:
+		print str(e)
+                print "Sending status failed..."
+                update = mesos_pb2.TaskStatus()
+                update.task_id.value = task.task_id.value
+                update.state = mesos_pb2.TASK_FAILED
+                update.data = 'data with a \0 byte' 
+                driver.sendStatusUpdate(update)
+                print "Sent status failed"
+                sys.exit(1) 
+            print "Sending status finished..."
+            update = mesos_pb2.TaskStatus()
+            update.task_id.value = task.task_id.value
+            update.state = mesos_pb2.TASK_FINISHED
+            update.data = 'data with a \0 byte'
+            driver.sendStatusUpdate(update)
+            print "Sent status finished"
+            sys.exit(0)
+	else:
+            # in parent 
+            print "Running task %s" % task.task_id.value
+            update = mesos_pb2.TaskStatus()
+            update.task_id.value = task.task_id.value
+            update.state = mesos_pb2.TASK_RUNNING
+            update.data = 'data with a \0 byte'
+            driver.sendStatusUpdate(update)
+            time.sleep(20)
+        '''
+    def frameworkMessage(self, driver, message):
+        # Send it back to the scheduler.
+        print "send message"
+        driver.sendFrameworkMessage(message)
+
+if __name__ == "__main__":
+    print "Starting executor on slave"
+    driver = mesos.native.MesosExecutorDriver(MyExecutor())
+    sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)