Groonga - fulltext search engine.

[Groonga-commit] groonga/gcs-console [master] Support upload of SDF batch #16

Yoji SHIDARA null+****@clear*****
Thu Nov 15 15:00:40 JST 2012

• Previous message: [Groonga-commit] groonga/gcs [master] Update scenario format for the latest runner
• Next message: [Groonga-commit] [Broken] groonga/gcs-console#25 (master - c17b0f7)
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

Yoji SHIDARA	2012-11-15 15:00:40 +0900 (Thu, 15 Nov 2012)

  New Revision: c17b0f7a93496807ae435bf85d01e4bbbda644da
  https://github.com/groonga/gcs-console/commit/c17b0f7a93496807ae435bf85d01e4bbbda644da

  Log:
    Support upload of SDF batch #16

  Added files:
    test/create-and-search-domain.test.js
    test/fixtures/add.sdf.json
    views/domain-upload.jade
  Modified files:
    app.js
    routes/index.js
    views/domain.jade

  Modified: app.js (+2 -0)
===================================================================
--- app.js    2012-11-13 17:10:37 +0900 (4cdc304)
+++ app.js    2012-11-15 15:00:40 +0900 (9ee899d)
@@ -43,6 +43,8 @@ function setupApplication(app) {
   app.get('/domain/:name', auth, routes.domain);
   app.get('/domain/:name/search', auth, routes.domainSearch);
   app.get('/domain/:name/index_fields', auth, routes.domainIndexFields);
+  app.get('/domain/:name/upload', auth, routes.domainUpload);
+  app.post('/domain/:name/upload', auth, routes.domainUploadPost);
   app.get('/domain_create', auth, routes.domainCreate);
   app.post('/domain_create', auth, routes.domainCreatePost);
   app.post('/domain/:name/index_fields', auth, routes.domainCreateIndexField);

  Modified: routes/index.js (+74 -0)
===================================================================
--- routes/index.js    2012-11-13 17:10:37 +0900 (300aabc)
+++ routes/index.js    2012-11-15 15:00:40 +0900 (9fc83bd)
@@ -1,16 +1,49 @@
 var querystring = require('querystring');
 var _ = require('underscore');
 var http = require('http');
+var awssum = require('awssum');
+var DocumentService = awssum.load('amazon/cloudsearch').DocumentService;
+var url = require('url');
+var fs = require('fs');
 
 exports.index = function(req, res) {
   res.render('index', {action: 'index'});
 };
 
+function countBytes(string) {
+  string = encodeURIComponent(string);
+  var escapedPartsMatcher = /\%[0-9a-f][0-9a-f]/gi;
+  var escapedParts = string.match(escapedPartsMatcher);
+  var notEscapedParts = string.replace(escapedPartsMatcher, '');
+  return notEscapedParts.length + (escapedParts ? escapedParts.length : 0);
+}
+
+function createGcsDocumentService(domain) {
+  var documentService = new DocumentService({
+    domainName: domain.DomainName,
+    domainId: domain.DomainId
+  });
+  var endpoint = url.parse('http://' + domain.DocService.Endpoint);
+  documentService.host = function() {
+    return endpoint.hostname;
+  };
+  documentService.addExtras = function(options, args) {
+    options.protocol = endpoint.protocol;
+    options.port = endpoint.port;
+  };
+  documentService.addCommonOptions = function(options, args) {
+    options.headers['content-type'] = 'application/json';
+    options.headers['content-length'] = countBytes(JSON.stringify(args.Docs));
+  };
+  return documentService;
+}
+
 function withDomain(req, res, callback) {
   var domain = _.where(res.locals.domains, {DomainName: req.params.name})[0];
   domain.isSelected = true;
 
   req.domain = domain;
+  req.documentService = createGcsDocumentService(domain);
 
   // get index fields
   req.cloudsearch.DescribeIndexFields({
@@ -278,3 +311,44 @@ exports.domainDeleteIndexField = function(req, res) {
     });
   });
 };
+
+exports.domainUpload = function(req, res) {
+  withDomain(req, res, function(req, res) {
+    res.render('domain-upload', {
+      action: 'domain_upload',
+      domain: req.domain
+    });
+  });
+};
+
+exports.domainUploadPost = function(req, res) {
+  withDomain(req, res, function(req, res) {
+    fs.readFile(req.files.batch.path, function(err, data) {
+      try {
+        var docs = JSON.parse(data);
+      } catch(e) {
+        res.status(400);
+        res.render('domain-upload', {
+          action: 'domain_upload',
+          domain: req.domain,
+          error: new Error('Failed to parse JSON')
+        });
+      }
+      var options = { Docs: docs };
+
+      req.documentService.DocumentsBatch(options, function(err, data) {
+        if (err) {
+          res.status(400);
+          res.render('domain-upload', {
+            action: 'domain_upload',
+            domain: req.domain,
+            error: errorToRender(err)
+          });
+        }
+
+        req.flash('info', 'SDF Batch Uploaded');
+        res.redirect('/domain/' + req.domain.DomainName + '/upload');
+      });
+    });
+  });
+};

  Added: test/create-and-search-domain.test.js (+82 -0) 100644
===================================================================
--- /dev/null
+++ test/create-and-search-domain.test.js    2012-11-15 15:00:40 +0900 (30e1eac)
@@ -0,0 +1,82 @@
+var assert = require('chai').assert;
+var Browser = require('zombie');
+var Target = require('./test-utils').Target;
+
+suite('dashboard', function() {
+  var target = new Target();
+  setup(function(done) {
+    target.setup(done)
+  });
+  teardown(function() {
+    target.teardown()
+  });
+
+  test('Create, upload and search', function(done) {
+    var browser = new Browser();
+    browser
+      .visit(target.rootURL)
+      .then(function() {
+        return browser.clickLink('Create New Domain');
+      })
+      .then(function() {
+        browser.fill('domain_name', 'companies');
+        return browser.pressButton('Create')
+      })
+      .then(function() {
+        assert.equal(browser.text('.alert'), 'Domain successfully created');
+        assert.equal(browser.location.pathname, '/domain/companies');
+      })
+      .then(function() {
+        return browser.clickLink('Index Fields');
+      })
+      .then(function() {
+        assert.equal(browser.location.pathname, '/domain/companies/index_fields');
+        assert.equal(browser.text('.alert-warn'), 'No IndexField is defined');
+      })
+      .then(function() {
+        browser.fill('name', 'name');
+        return browser.pressButton('Create')
+      })
+      .then(function() {
+        browser.fill('name', 'address');
+        return browser.pressButton('Create')
+      })
+      .then(function() {
+        browser.fill('name', 'email_address');
+        return browser.pressButton('Create')
+      })
+      .then(function() {
+        browser.fill('name', 'age');
+        return browser.pressButton('Create')
+      })
+      .then(function() {
+        browser.fill('name', 'product');
+        return browser.pressButton('Create')
+      })
+      .then(function() {
+        return browser.clickLink('Upload');
+      })
+      .then(function() {
+        assert.equal(browser.text('h2'), 'Upload SDF Batch');
+      })
+      .then(function() {
+        browser.attach('batch', __dirname + '/fixtures/add.sdf.json');
+        return browser.pressButton('Upload');
+      })
+      .then(function() {
+        assert.equal(browser.text('.alert'), 'SDF Batch Uploaded');
+      })
+      .then(function() {
+        return browser.clickLink('Search');
+      })
+      .then(function() {
+        // Zombie.js does not handle GET form at this time.
+        // So hit the search result page directly.
+        return browser.visit('/domain/companies/search?query=tokyo');
+      })
+      .then(function() {
+        assert.equal(browser.text('.alert-info'), 'Found 3 records. Showing 1 - 3 (3 records).');
+      })
+      .then(done, done);
+  });
+});

  Added: test/fixtures/add.sdf.json (+125 -0) 100644
===================================================================
--- /dev/null
+++ test/fixtures/add.sdf.json    2012-11-15 15:00:40 +0900 (03b4b9d)
@@ -0,0 +1,125 @@
+[
+  {
+    "type": "add",
+    "id": "id1",
+    "version": 1,
+    "lang": "en",
+    "fields": {
+      "name": "Brazil",
+      "address": "Shibuya, Tokyo, Japan",
+      "email_address": "info �� razil.jp",
+      "age": 1,
+      "product": "groonga"
+    }
+  },
+  {
+    "type": "add",
+    "id": "id2",
+    "version": 1,
+    "lang": "en",
+    "fields": {
+      "name": "Enishi Tech Inc.",
+      "address": "Sapporo, Hokkaido, Japan",
+      "email_address": "info �� enishi-tech.com",
+      "age": 2,
+      "product": "groonga"
+    }
+  },
+  {
+    "type": "add",
+    "id": "id3",
+    "version": 1,
+    "lang": "en",
+    "fields": {
+      "name": "ClearCode Inc.",
+      "address": "Hongo, Tokyo, Japan",
+      "email_address": "info �� clear-code.com",
+      "age": 3,
+      "product": "groonga"
+    }
+  },
+  {
+    "type": "add",
+    "id": "id4",
+    "version": 1,
+    "lang": "en",
+    "fields": {
+      "name": "Anaheim Electronics",
+      "address": "Granada, Moon",
+      "age": 4,
+      "product": "gundam"
+    }
+  },
+  {
+    "type": "add",
+    "id": "id5",
+    "version": 1,
+    "lang": "en",
+    "fields": {
+      "name": "Shinsei Industry",
+      "address": "Earth",
+      "age": 5,
+      "product": "valkyrie"
+    }
+  },
+  {
+    "type": "add",
+    "id": "id6",
+    "version": 1,
+    "lang": "en",
+    "fields": {
+      "name": "Omni Consumer Products",
+      "address": "Detroit and Delta City, Michigan, United States",
+      "age": 6,
+      "product": "robocop"
+    }
+  },
+  {
+    "type": "add",
+    "id": "id7",
+    "version": 1,
+    "lang": "en",
+    "fields": {
+      "name": "Capsule Corporation",
+      "address": "West City",
+      "age": 7,
+      "product": "time machine"
+    }
+  },
+  {
+    "type": "add",
+    "id": "id8",
+    "version": 1,
+    "lang": "en",
+    "fields": {
+      "name": "Stark Industries",
+      "address": "United States",
+      "age": 8,
+      "product": "iron man"
+    }
+  },
+  {
+    "type": "add",
+    "id": "id9",
+    "version": 1,
+    "lang": "en",
+    "fields": {
+      "name": "Umbrella Corporation",
+      "address": "Tokyo, Japan",
+      "age": 9,
+      "product": "tyrant"
+    }
+  },
+  {
+    "type": "add",
+    "id": "id10",
+    "version": 1,
+    "lang": "en",
+    "fields": {
+      "name": "U.S. Robots and Mechanical Men",
+      "address": "New York, United States",
+      "age": 10,
+      "product": "ndr114"
+    }
+  }
+]

  Added: views/domain-upload.jade (+13 -0) 100644
===================================================================
--- /dev/null
+++ views/domain-upload.jade    2012-11-15 15:00:40 +0900 (f5990ab)
@@ -0,0 +1,13 @@
+extends domain
+
+block domain-content
+  h2 Upload SDF Batch
+
+  form(action="/domain/"+domain.DomainName+"/upload", method="post", enctype="multipart/form-data").form-horizontal
+    .control-group
+      label.control-label SDF Batch
+      .controls
+        input(type="file", name="batch")
+    .control-group
+      .controls
+        button.btn.btn-primary(type="submit") Upload

  Modified: views/domain.jade (+2 -0)
===================================================================
--- views/domain.jade    2012-11-13 17:10:37 +0900 (de6117d)
+++ views/domain.jade    2012-11-15 15:00:40 +0900 (b1add87)
@@ -10,5 +10,7 @@ block content
       a(href="/domain/"+domain.DomainName+"/search") Search
     li(class=(action == "domain_index_fields" ? "active" : ""))
       a(href="/domain/"+domain.DomainName+"/index_fields") Index Fields
+    li(class=(action == "domain_upload" ? "active" : ""))
+      a(href="/domain/"+domain.DomainName+"/upload") Upload
 
   block domain-content
-------------- next part --------------
HTML����������������������������...
Download 

• Previous message: [Groonga-commit] groonga/gcs [master] Update scenario format for the latest runner
• Next message: [Groonga-commit] [Broken] groonga/gcs-console#25 (master - c17b0f7)
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]