susumu.yata
null+****@clear*****
Tue Mar 24 22:08:35 JST 2015
susumu.yata 2015-03-24 22:08:35 +0900 (Tue, 24 Mar 2015) New Revision: 306d1e1d603af5cc8a5a4622c4ae9c5a0012e70e https://github.com/groonga/grnxx/commit/306d1e1d603af5cc8a5a4622c4ae9c5a0012e70e Message: Gnx: add a new benchmark. Added files: go/benchmark2.go Added: go/benchmark2.go (+202 -0) 100644 =================================================================== --- /dev/null +++ go/benchmark2.go 2015-03-24 22:08:35 +0900 (2f2e48d) @@ -0,0 +1,202 @@ +package main + +import "./gnx" + +import "bufio" +import "encoding/json" +import "flag" +import "fmt" +import "log" +import "math/rand" +import "os" +import "runtime" +import "runtime/pprof" +import "strconv" +import "strings" +import "time" +import "unicode" + +var flagMode = flag.String("mode", "run", "mode (run or print)") +var flagLoad = flag.String("load", "load", "load type (load or gnx_load)") +var flagKey = flag.String("key", "", "key data type") +var flagColumns = flag.String("columns", "Bool,Int16,Float,Text", + "comma-separated column data types") +var flagRows = flag.Int("rows", 1000000, "number of rows") +var flagBlock = flag.Int("block", 10, "number of rows per command") +var flagPartition = flag.Int("partition", 1, "number of groonga DBs") +var flagCPU = flag.Int("cpu", 1, "number of CPU cores") +var flagProfile = flag.String("profile", "benchmark.prof", + "path of profile data") + +func generateValue(typeName string) interface{} { + switch typeName { + case "Bool": + return (rand.Int() % 2) == 1 + case "Int8": + return int8(rand.Int() % 128) + case "Int16": + return int16(rand.Int() % 32768) + case "Int32": + return rand.Int31() + case "Int64": + return rand.Int63() + case "UInt8": + return uint8(rand.Int() % 256) + case "UInt16": + return uint16(rand.Int() % 65536) + case "UInt32": + return rand.Uint32() + case "UInt64": + return uint64(rand.Int63()) + case "Float": + return rand.Float64() + case "ShortText", "Text", "LongText": + return strconv.Itoa(int(rand.Int31())) + default: + log.Fatalln("unsupported data type: name =", typeName) + return nil + } +} + +func generateCommands() []string { + var commands []string + if *flagKey == "" { + commands = append(commands, "table_create Table TABLE_NO_KEY") + } else { + commands = append(commands, + fmt.Sprintf("table_create Table TABLE_PAT_KEY %s", *flagKey)) + } + + var columnTypes []string + var columnNames []string + if *flagColumns != "" { + tokens := strings.Split(*flagColumns, ",") + for _, token := range tokens { + columnType := strings.TrimFunc(token, unicode.IsSpace) + if columnType != "" { + columnID := len(columnTypes) + columnTypes = append(columnTypes, columnType) + columnName := fmt.Sprintf("Col%d", columnID) + columnNames = append(columnNames, columnName) + commands = append(commands, + fmt.Sprintf("column_create Table %s COLUMN_SCALAR %s", + columnName, columnType)) + switch columnType { + case "Text", "LongText": + const tokenizerOption = "--default_tokenizer TokenBigram" + commands = append(commands, fmt.Sprintf( + "table_create Idx%d TABLE_PAT_KEY ShortText %s", + columnID, tokenizerOption)) + commands = append(commands, fmt.Sprintf( + "column_create Idx%d Idx COLUMN_INDEX|WITH_POSITION Table Col%d", + columnID, columnID)) + } + } + } + } + + columnsOption := strings.Join(columnNames, ",") + numValues := len(columnNames) + if *flagKey != "" { + if columnsOption == "" { + columnsOption = "_key" + } else { + columnsOption = "_key," + columnsOption + } + numValues += 1 + } + loadPrefix := fmt.Sprintf("%s --table Table --columns '%s' --values ", + *flagLoad, columnsOption) + scanner := bufio.NewScanner(os.Stdin) + for i := 0; i < *flagRows; i += *flagBlock { + var blockValues [][]interface{} + for j := 0; (j < *flagBlock) && ((i + j) < *flagRows); j++ { + var rowValues []interface{} + if *flagKey != "" { + rowValues = append(rowValues, generateValue(*flagKey)) + } + for _, columnType := range columnTypes { + switch columnType { + case "Text", "LongText": + if scanner.Scan() { + rowValues = append(rowValues, scanner.Text()) + } else { + rowValues = append(rowValues, generateValue(columnType)) + } + default: + rowValues = append(rowValues, generateValue(columnType)) + } + } + blockValues = append(blockValues, rowValues) + } + bytes, err := json.Marshal(blockValues) + if err != nil { + log.Fatalln(err) + } + block := string(bytes) + block = strings.Replace(block, "\\", "\\\\", -1) + block = strings.Replace(block, "'", "\\'", -1) + commands = append(commands, loadPrefix+"'"+block+"'") + } + return commands +} + +func runCommands(commands []string) { + file, err := os.Create(*flagProfile) + if err != nil { + log.Fatalln(err) + } + defer file.Close() + if err := pprof.StartCPUProfile(file); err != nil { + log.Fatalln(err) + } + defer pprof.StopCPUProfile() + + db, dir, err := gnx.CreateTempDB("", "benchmark", *flagPartition) + if err != nil { + log.Fatalln(err) + } + defer os.RemoveAll(dir) + defer db.Close() + startTime := time.Now() + for _, command := range commands { + if _, err := db.Query(command); err != nil { + log.Fatalln(err) + } + } + endTime := time.Now() + fmt.Println("elapsed:", endTime.Sub(startTime)) +} + +func printCommands(commands []string) { + db, dir, err := gnx.CreateTempDB("", "benchmark", *flagPartition) + if err != nil { + log.Fatalln(err) + } + defer os.RemoveAll(dir) + defer db.Close() + for _, command := range commands { + fmt.Println(command) +// if _, err := db.Query(command); err != nil { +// log.Fatalln(err) +// } + } +} + +func main() { + flag.Parse() + if *flagCPU == 0 { + runtime.GOMAXPROCS(runtime.NumCPU()) + } else { + runtime.GOMAXPROCS(*flagCPU) + } + commands := generateCommands() + switch *flagMode { + case "run": + runCommands(commands) + case "print": + printCommands(commands) + default: + log.Fatalln("undefined mode") + } +} -------------- next part -------------- HTML����������������������������...Download