misc/data/datagen.c

/*
  Random Japanese data generator

  @author Tetsuro IKEDA <ikdttr@gmail.com>

  Copyright(c) 2008 Tritonn Project
  License LGPL v2
*/

#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/time.h>

/* number of parses */
#define PARSE_NUM 25

typedef struct st_dict_info {
    char *fname;
    int raito;
} DICT_INFO;

DICT_INFO dict_info[] = {
  {"Noun.others.csv",   10},
  {"Noun.verbal.csv",   10},
  {"Symbol.csv",        10},
  {"Noun.demonst.csv",  10},
  {"Interjection.csv",  10},
  {"Prefix.csv",        10},
  {"Noun.adjv.csv",     10},
  {"Conjunction.csv",   80},
  {"Noun.place.csv",    10},
  {"Adj.csv",           30},
  {"Noun.nai.csv",      10},
  {"Noun.proper.csv",   10},
  {"Postp-col.csv",     10},
  {"Noun.csv",          80},
  {"Noun.org.csv",      10},
  {"Others.csv",         1},
  {"Filler.csv",        10},
  {"Adnominal.csv",     10},
  {"Auxil.csv",         10},
  {"Verb.csv",          60},
  {"Adverb.csv",        30},
  {"Noun.number.csv",   10},
  {"Suffix.csv",        10},
  {"Postp.csv",         10},
  {"Noun.name.csv",     30},
  {"Noun.adverbal.csv", 10}
};

int row_num[] = {
  151,
  12146,
  208,
  120,
  252,
  221,
  3328,
  171,
  72999,
  27210,
  42,
  27327,
  91,
  60477,
  16668,
  2,
  19,
  135,
  199,
  130750,
  3032,
  42,
  1393,
  146,
  34202,
  795
};

typedef struct st_parse {
  char *name;
  int row_num;
  int *row_size;
  char (*data)[128];
} PARSE;

PARSE dict[PARSE_NUM];

void print_parse(PARSE *p)
{
    int i;
    for (i=0; i < p->row_num; i++) {
        printf("name=%s, row_num=%d, row_id=%d, row_size=%d, data=%s\n",
            p->name,p->row_num,i+1,p->row_size[i],p->data[i]);
    }   
}

void print_dict(PARSE *p)
{
  int i;
  for (i=0; i < PARSE_NUM; i++) {
    print_parse(p++);
  }
}
/*
  read .csv file into parse

  PARSE *p : target parse to be loaded
             this must have "name" as file name.
  return   : 0 success -1 failed
*/
int load_parse(PARSE *p)
{ 
  if (!p || !(p->name)) {
    printf("load_parse: incorrect argument\n");
    return -1;
  }
  FILE *f;
  if (f = fopen(p->name, "r")) {
    if (!(p->row_size = malloc((sizeof p->row_size) * p->row_num))) {
      printf("malloc error\n");
      return -1;
    }
    int msize = p->row_num * 128;
    if (!(p->data = malloc(msize))) {
          printf("cannot malloc for file '%s', malloc size=%d\n",msize);
          return -1;
    }
    int line;
    char *res;
    for (line=0; line < p->row_num; line++) {
        if(!(res = fgets(p->data[line],128,f))) {
            printf("cannot read expected rows from %s, currently #%d\n",p->name,line+1);
            return -1;
        }
        int i;
        /* replace '\n' into '\0' */
        for (i=0; i <= 128; i++) {
            if ((p->data[line][i] == '\n') && (p->data[line][i+1] == '\0')) {
                p->data[line][i] = '\0';
                p->row_size[line] = i;
                break;
            }
        }
    }
  } else {
    printf("cannot open file '%s'\n",p->name);
    return -1;
  }
  fclose(f);
  return 0;
}

int raito_sum=0;
int dice_parse_id()
{
    int i;
    struct timeval t;
    gettimeofday(&t,NULL);
    int seed = (int) t.tv_usec;
    /* cache raito_sum */
    if (raito_sum == 0) {
        for (i=0; i < PARSE_NUM; i++) raito_sum+=dict_info[i].raito;
    }
    int rand_id = rand_r(&seed);
    rand_id = rand_id % raito_sum;
    int curr_id=0;
    for (i=0; i < PARSE_NUM; i++) {
        curr_id += dict_info[i].raito;
        if (curr_id > rand_id) {
            break;
        } else {
            
        }
    }
    return i;
}

int dice_row_id(int dict_id) 
{
    struct timeval t;
    gettimeofday(&t,NULL);
    int seed = (int) t.tv_usec;
    int rand_id = rand_r(&seed);
    return rand_id % dict[dict_id].row_num;
}

int main(int argc, char *argv[])
{
    if (argc != 3) {
        printf("usage: datagen2 [row_size] [row_num]\n");
        return 0;
    }
    int gen_row_size=atoi(argv[1]);
    int gen_row_num=atoi(argv[2]);
    char *buf;
    
    int i;

    /* init and load dictionaries */
    for (i=0; i < PARSE_NUM; i++) {
        dict[i].name = dict_info[i].fname;
        dict[i].row_num = row_num[i];
        if (load_parse(&dict[i])) {
            printf("parse load error\n");
            return -1;
        }
    }
    
    int offset;
    for (i=0; i < gen_row_num; i++) {
        buf = malloc(gen_row_size+1);
        offset = 0;
        while (1) {
            int dict_id = dice_parse_id();
            int row_id = dice_row_id(dict_id);
            int word_size = dict[dict_id].row_size[row_id];
            if (offset + word_size > gen_row_size) break;
            int j;
            for (j=0; j < word_size; j++) {
                buf[offset+j] = dict[dict_id].data[row_id][j];
            }
            offset += word_size;
        }
        printf("%s\n",buf);
        free(buf);
    } 
    return 0;
}
