【CNTK/OpenCV/Android】Server+Android+CNN實現移動端圖像識別系統

使用CNTK訓練的網絡，和OpenCV提供的圖片處理支持。在Android平台將圖片上傳服務器並返回圖片種類。

留自己備忘。

首先是第一個BUG： 用NDK太麻煩，所以講CNTK和OpenCV接口直接編譯好了一個exe，用Java Runtime直接調用。但是除了一個問題，花了半天時間。原因是CNTK函數返回了很多日志信息，這些日志信息如果不讀出來，超過JVM分配的緩沖區大小，就會造成程序阻塞。產生的效果就是，雖然已經調用了exe，但是該exe只有在Java程序執行結束後，才會繼續執行。

---

第二個BUG： Android端經過剪裁處理後的圖片上傳有白邊。這個原因雖然解決了，但是理解的還不是很好。因為在進行Crop處理時，需要傳進要裁剪圖片size的大小，如果你裁剪的大小超過這個size，那麼沒事，如果小於這個size，那麼會以白邊填充。這樣傳到服務器的圖片也有白邊，這基本就識別不准了。

---

Eval.cpp

#include 
#include 
#include "Eval.h"
#ifdef _WIN32
#include "Windows.h"
#endif
#include    
#include   
using namespace Microsoft::MSR::CNTK;
using namespace std;

template
using GetEvalProc = void(*)(IEvaluateModel**);

typedef std::pair*> MapEntry;
typedef std::map*> Layer;

int main(int argc, char* argv[])
{

    argc = 0;
    std::string app = "D:/cntk/Examples/Image/";
    std::string path;
    ofstream myfile("D:/get.txt", ios::out);
    char *imagevector = argv[1];

    IEvaluateModel *model;

#ifdef _WIN32
    path = app.substr(0, app.rfind("\\"));


    const std::string modelWorkingDirectory = path + "/../../Examples/Image/MyTestDNN/Data/";
#else // on Linux
    path = app.substr(0, app.rfind("/"));

    // This relative path assumes launching from CNTK's binary folder, e.g. build/release/bin/
    const std::string modelWorkingDirectory = path + "/../../../Examples/Image/MNIST/Data/";
#endif

    GetEvalF(&model);

    const std::string modelFilePath = modelWorkingDirectory + "../Output/Models/02_Convolution";

    std::string networkConfiguration;

    networkConfiguration += "modelPath=\"" + modelFilePath + "\"";
    model->CreateNetwork(networkConfiguration);

    std::map inDims;
    std::map outDims;
    model->GetNodeDimensions(inDims, NodeGroup::nodeInput);
    model->GetNodeDimensions(outDims, NodeGroup::nodeOutput);

    auto inputLayerName = inDims.begin()->first;
    std::vector inputs;

    //讀取輸入進來的向量值（以“，”分割的）
    char *p = NULL;
    p = strtok(imagevector, ",");

    float f = atof(p);
    inputs.push_back(f);
    while ((p = strtok(NULL, ",")) != NULL)
    {
        f = atof(p);
        inputs.push_back(f);

    }

    std::vector outputs;
    Layer inputLayer;
    inputLayer.insert(MapEntry(inputLayerName, &inputs));
    Layer outputLayer;
    auto outputLayerName = outDims.begin()->first;
    outputLayer.insert(MapEntry(outputLayerName, &outputs));
    model->Evaluate(inputLayer, outputLayer);
    for (auto& value : outputs)
    {

        cout << value << endl;
        myfile << value <<",";
        myfile.flush();
    }

    return 0;
}

Vector.cpp

#include 
#include 
#include
#include  
#include    
#include 

using   namespace   std;
using   namespace   cv;

int main(int argc, char* argv[])
{
    char *classimage = argv[1];
    IplImage *image, *imageresize = 0;
    image = cvLoadImage(classimage, 1);
    imageresize = cvCreateImage(cvSize(28, 28), IPL_DEPTH_8U, 3);
    cvResize(image, imageresize, CV_INTER_LINEAR);
    IplImage* gray = cvCreateImage(cvGetSize(imageresize), IPL_DEPTH_8U, 1);//用原圖像指針創建新圖像
    cvCvtColor(imageresize, gray, CV_BGR2GRAY);

    //提取向量
    stringstream ss;
    string s;
    ofstream myfile("D:\\vector.txt", ios::out);
    for (int m = 0; m < gray->height; m++){
        for (int j = 0; j < gray->width; j++){
            //獲取灰度圖的像素
            CvScalar dPixelVal = cvGet2D(gray, m, j);
            char temp[20];
            sprintf(temp, "%d", (int)dPixelVal.val[0]);
            s.append(temp);
            s.append(",");
        }
    }

    s = s.substr(0, s.length() - 1);
    //輸出向量
    myfile << s << endl;
    cout << s << endl;
    cvReleaseImage(&image);
    cvReleaseImage(&imageresize);
    system("pause");
    return 0;
}

服務器端

Servlet

package com.sunyang.servlet;

public class UploadServlet extends HttpServlet {

    private static final long serialVersionUID = 1L;
    String temp = new File("").getAbsolutePath();   
    String webappPath = temp.replace("bin", "webapps\\Image");
    public void doGet(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {

        doPost(request, response);
    }

    public void doPost(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        String ImageToVector = "";
        String CNTK03 = "";
        request.setCharacterEncoding("utf-8");
        response.setCharacterEncoding("utf-8");
        response.setContentType("text/html;charset=utf-8");
        PrintWriter out = response.getWriter();
        SmartUpload smartUpload = new SmartUpload();
        String msg=request.getParameter("msg");
//      out.print(msg);  
        try {  
            smartUpload.initialize(this.getServletConfig(), request, response); 
            smartUpload.upload();  
            com.jspsmart.upload.File smartFile = smartUpload.getFiles().getFile(0);  
            if (!smartFile.isMissing()) { 
                String saveFileName = "images/" + smartFile.getFileName();  
                smartFile.saveAs(saveFileName, SmartUpload.SAVE_VIRTUAL);  
                String path = webappPath+File.separator+saveFileName;
                String imagePath = path.replace("\\", "/");
                System.out.println(imagePath);
                String vector = GetModel.transImage(ImageToVector, imagePath);
                int family = GetModel.GetResult(CNTK03, vector);
                System.out.println(family);
                out.print("The plant is:"+family+"!"); 

            } else {  
                out.print("missing...");  
            }  
        } catch (Exception e) {  
            out.print(e+","+msg);  
        } 
        out.flush();
        out.close();
    }

}

GetModel

package com.sunyang.servlet;

public class GetModel {

    public static String transImage(String cmd, String path) {
        String command[] = new String[] { cmd, path };
        String vector=null;
        try {
            Runtime.getRuntime().exec(command);
            Thread.sleep(1000);
            // 讀文件
                        try {
                            String encoding = "GBK";
                            File file = new File("D:/vector.txt");
                            if (file.isFile() && file.exists()) { // 判斷文件是否存在
                                InputStreamReader read = new InputStreamReader(
                                        new FileInputStream(file), encoding);// 考慮到編碼格式
                                BufferedReader bufferedReader = new BufferedReader(read);
                                String lineTxt = null;
                                while ((lineTxt = bufferedReader.readLine()) != null) {
                                    System.out.println(lineTxt);
                                    vector = lineTxt;
                                }
                                read.close();
                            } else {
                                System.out.println("找不到指定的文件");
                            }
                        } catch (Exception e) {
                            System.out.println("讀取文件內容出錯");
                            e.printStackTrace();
                        }

        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return vector;
    }

    public static int GetResult(String cmd, String vector) {
        int result=-1;
        final Process process;

        try {
            String cmdStr = "cmd /c "+cmd+" "+vector ;
            process = Runtime.getRuntime().exec(cmdStr);
            printMessage(process.getInputStream());
            printMessage(process.getErrorStream());
            Thread.sleep(1000);
            // 讀文件
            try {
                File file = new File("D:/get.txt");
                if (file.isFile() && file.exists()) { // 判斷文件是否存在
                    InputStreamReader read = new InputStreamReader(
                            new FileInputStream(file));// 考慮到編碼格式
                    BufferedReader bufferedReader = new BufferedReader(read);
                    String lineTxt = null;
                    lineTxt = bufferedReader.readLine();
                    System.out.println(lineTxt);
                    result = GetFamily(lineTxt);
                    read.close();

                } else {
                    System.out.println("找不到指定的文件");
                }
            } catch (Exception e) {
                System.out.println("讀取文件內容出錯");
                e.printStackTrace();
            }

        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return result;
    }

    //查找並排序
    public static int GetFamily(String data){
        //去掉data最後一個“，”
        String myData = data.substring(0, data.length()-1);
        String[] split = myData.split(",");
        final Hashtable ht = new Hashtable();
        int count = 1;
        for (int i = 0; i < split.length; i++) {
            ht.put(Double.parseDouble(split[i]),count);
            count++;
        }
        List list = new ArrayList(ht.keySet());
        Collections.sort(list);
        int result = ht.get(list.get(ht.size()-1));
        System.out.println("輸出類別"+result);
        return result;
    }
}

Android端

MainActivity

package com.sunyang.Image;
public class MainActivity extends Activity {

    private ImageView img;
    private Button btnUpload;
    private HttpUtils httpUtils;
    private String URL="http://192.168.1.119:8080/Person_proj/upload";
    private String[] items = { "打開相機", "從相冊選擇" };
    private String title = "樹葉識別";

    private static final int PHOTO_CARMERA = 1;
    private static final int PHOTO_PICK = 2;
    private static final int PHOTO_CUT = 3;
    private File tempFile = new File(Environment.getExternalStorageDirectory(),
            getPhotoFileName());

    @SuppressLint("SimpleDateFormat") private String getPhotoFileName() {
        Date date = new Date(System.currentTimeMillis());
        SimpleDateFormat sdf = new SimpleDateFormat("'PNG'_yyyyMMdd_HHmmss");

        return sdf.format(date) + ".png";
    }

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        img = (ImageView) findViewById(R.id.main_img);
        btnUpload = (Button) findViewById(R.id.upload);
        btnUpload.setOnClickListener(clickListener);
        img.setOnClickListener(clickListener);

        httpUtils=new HttpUtils(10000);
    }

    private OnClickListener clickListener = new OnClickListener() {

        @Override
        public void onClick(View v) {
            switch (v.getId()) {
            case R.id.main_img:
                AlertDialog.Builder dialog = AndroidUtil.getListDialogBuilder(
                        MainActivity.this, items, title, dialogListener);
                dialog.show();
                break;
            case R.id.upload:
                upload();
                break;

            default:
                break;
            }

        }
    };

    //上傳
    protected void upload() {
        RequestParams params=new RequestParams();
        params.addBodyParameter(tempFile.getPath().replace("/", ""), tempFile);
        httpUtils.send(HttpMethod.POST,URL, params,new RequestCallBack() {

            @Override
            public void onFailure(HttpException e, String msg) {
                Toast.makeText(MainActivity.this, "上傳成功", Toast.LENGTH_SHORT).show();
                Log.i("MainActivity", e.getExceptionCode() + "====="
                        + msg);
            }

            @Override
            public void onSuccess(ResponseInfo responseInfo) {
                Log.i("MainActivity", "====upload_success====="
                        + responseInfo.result);
                Toast.makeText(MainActivity.this, responseInfo.result,Toast.LENGTH_LONG).show();
            }
        });
    }

    private android.content.DialogInterface.OnClickListener dialogListener = new DialogInterface.OnClickListener() {

        @Override
        public void onClick(DialogInterface dialog, int which) {
            switch (which) {
            case 0:
                // 彈出對話框
                startCamera(dialog);
                break;
            case 1:
                // 從相冊選
                startPick(dialog);
                break;

            default:
                break;
            }
        }
    };

    // 打開相機
    protected void startCamera(DialogInterface dialog) {
        dialog.dismiss();
        Intent intent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE);
        intent.putExtra("camerasensortype", 2); 
        intent.putExtra("autofocus", true); 
        intent.putExtra("fullScreen", false); 
        intent.putExtra("showActionIcons", false);
        intent.putExtra(MediaStore.EXTRA_OUTPUT, Uri.fromFile(tempFile));
        startActivityForResult(intent, PHOTO_CARMERA);
    }

    // 找相冊裡的
    protected void startPick(DialogInterface dialog) {
        dialog.dismiss();
        Intent intent = new Intent(Intent.ACTION_PICK, null);
        intent.setDataAndType(MediaStore.Images.Media.EXTERNAL_CONTENT_URI,
                "image/*");
        startActivityForResult(intent, PHOTO_PICK);
    }

    @Override
    protected void onActivityResult(int requestCode, int resultCode, Intent data) {
        switch (requestCode) {
        case PHOTO_CARMERA:
            startPhotoZoom(Uri.fromFile(tempFile), 100);
            break;
        case PHOTO_PICK:
            if (null != data) {
                startPhotoZoom(data.getData(), 100);
            }
            break;
        case PHOTO_CUT:
            if (null != data) {
                setPicToView(data);
            }
            break;

        default:
            break;
        }
        super.onActivityResult(requestCode, resultCode, data);
    }

    // 縮放圖片
    private void startPhotoZoom(Uri uri, int size) {
        Intent intent = new Intent("com.android.camera.action.CROP");
        intent.setDataAndType(uri, "image/*");
        intent.putExtra("crop", true);
        // 比例寬高
        intent.putExtra("aspectX", 1);
        intent.putExtra("aspectY", 1);
        // 剪裁大小
        intent.putExtra("outputX", size);
        intent.putExtra("outputY", size);
        intent.putExtra("return-data", true);
        startActivityForResult(intent, PHOTO_CUT);
    }

    //把圖片加載到view上
    private void setPicToView(Intent data) {
        Bundle bundle = data.getExtras();
        if (null != bundle) {
            final Bitmap bmp = bundle.getParcelable("data");
            img.setImageBitmap(bmp);
            saveCropPic(bmp);
            Log.i("MainActivity", tempFile.getAbsolutePath());
        }
    }

    // 圖片保存到sd卡
    private void saveCropPic(Bitmap bmp) {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        FileOutputStream fis = null;
        bmp.compress(Bitmap.CompressFormat.PNG, 100, baos);
        try {
            fis = new FileOutputStream(tempFile);
            fis.write(baos.toByteArray());
            fis.flush();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                if (null != baos) {
                    baos.close();
                }
                if (null != fis) {
                    fis.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

Layout