* scrapper.pl の screen-mobile-device が更新されていない問題を修正
trunk/maple/test/scripts/scrapper.pl (diff)| @@ -6,19 +6,20 @@ | ||
| 6 | 6 | no lib qw(.); |
| 7 | 7 | use Config::Tiny qw(); |
| 8 | 8 | use IO::String qw(); |
| 9 | +use LWP::Simple; | |
| 9 | 10 | use Net::CIDR::MobileJP::Scraper qw(); |
| 10 | 11 | use Text::CSV_XS qw(); |
| 11 | 12 | use Unicode::Japanese qw(); |
| 12 | 13 | |
| 13 | -my $CIDR_CONFIG_FILE = '../../img0ch/_system/CIDR-mobile-jp.ini'; | |
| 14 | -my $SCREEN_CONFIG_FILE = '../../img0ch/_system/screen-mobile-device.ini'; | |
| 14 | +my $CIDR_CONFIG_FILE = '../../img0ch/_system/CIDR-mobile-jp.ini'; | |
| 15 | +my $SCREEN_CONFIG_FILE = '../../img0ch/_system/screen-mobile-device.ini'; | |
| 15 | 16 | |
| 16 | 17 | CIDR: { |
| 17 | - my $cidr = Config::Tiny->new(); | |
| 18 | - my $result = Net::CIDR::MobileJP::Scraper->new()->run(); | |
| 18 | + my $cidr = Config::Tiny->new(); | |
| 19 | + my $result = Net::CIDR::MobileJP::Scraper->new()->run(); | |
| 19 | 20 | my @emobile = qw(117.55.1.224/27); |
| 20 | - my @ibis = qw(219.117.203.9); | |
| 21 | - my @pcsb = qw(123.108.237.240/28 202.253.96.0/28); | |
| 21 | + my @ibis = qw(219.117.203.9); | |
| 22 | + my @pcsb = qw(123.108.237.240/28 202.253.96.0/28); | |
| 22 | 23 | map { $cidr->{pcsb}->{$_} = 1 } sort @pcsb; |
| 23 | 24 | map { $cidr->{ibis}->{$_} = 1 } sort @ibis; |
| 24 | 25 | map { $cidr->{emobile}->{$_} = 1 } sort @emobile; |
| @@ -31,26 +32,31 @@ | ||
| 31 | 32 | } |
| 32 | 33 | |
| 33 | 34 | SCREEN: { |
| 34 | - my $lwp = LWP::UserAgent->new(); | |
| 35 | - my $res = $lwp->get('http://ke-tai.org/moblist/csv_down.php'); | |
| 36 | - my $csv = Text::CSV_XS->new({ binary => 1 }); | |
| 37 | - my $io = IO::String->new($res->content()); | |
| 38 | - do { <$io> }; | |
| 35 | + mirror( 'http://ke-tai.org/moblist/csv_down.php', 'ktai.csv' ); | |
| 36 | + my $csv = Text::CSV_XS->new( { binary => 1 } ) | |
| 37 | + or die Text::CSV->error_diag(); | |
| 38 | + open my $io, '<', 'ktai.csv' or die $!; | |
| 39 | + do {<$io>}; | |
| 39 | 40 | my $columns = <$io>; |
| 40 | - my $unijp = Unicode::Japanese->new(); | |
| 41 | - $unijp->set($columns, 'sjis'); | |
| 41 | + my $unijp = Unicode::Japanese->new(); | |
| 42 | + $unijp->set( $columns, 'sjis' ); | |
| 42 | 43 | my $columns_utf8 = $unijp->get(); |
| 43 | - my $screen = Config::Tiny->new(); | |
| 44 | - while (my $row = $csv->getline($io)) { | |
| 45 | - $unijp->set($row->[3]); | |
| 44 | + my $screen = Config::Tiny->new(); | |
| 45 | + | |
| 46 | + while ( my $line = <$io> ) { | |
| 47 | + $csv->parse($line); | |
| 48 | + my $row = [ $csv->fields() ]; | |
| 49 | + $unijp->set( $row->[3] ); | |
| 46 | 50 | my $carrier = lc $row->[1]; |
| 47 | - my $device = $unijp->get(); | |
| 48 | - my $width = $row->[7]; | |
| 49 | - my $height = $row->[8]; | |
| 50 | - ($device and $width and $height) or next; | |
| 51 | + my $device = $unijp->get(); | |
| 52 | + my $width = $row->[7]; | |
| 53 | + my $height = $row->[8]; | |
| 54 | + ( $device and $width and $height ) or next; | |
| 51 | 55 | $screen->{$carrier} ||= {}; |
| 52 | - if ($device =~ m{/}xms) { | |
| 53 | - map { $screen->{$carrier}->{$_} = join ',', $width, $height } split m{\s*/\s*}, $device; | |
| 56 | + | |
| 57 | + if ( $device =~ m{/}xms ) { | |
| 58 | + map { $screen->{$carrier}->{$_} = join ',', $width, $height } | |
| 59 | + split m{\s*/\s*}, $device; | |
| 54 | 60 | } |
| 55 | 61 | else { |
| 56 | 62 | $device =~ s/\A\s*//xms; |
| @@ -58,6 +64,7 @@ | ||
| 58 | 64 | $screen->{$carrier}->{$device} = join ',', $width, $height; |
| 59 | 65 | } |
| 60 | 66 | } |
| 67 | + close $io or die $!; | |
| 61 | 68 | $screen->write($SCREEN_CONFIG_FILE) or die $SCREEN_CONFIG_FILE, ': ', $!; |
| 62 | 69 | } |
| 63 | 70 |
hikarin