まずHatenaBookmarkUsersCount。interval追加しただけ。
1c1
< package Plagger::Plugin::Filter::HatenaBookmarkUsersCount;
---
> package Plagger::Plugin::Filter::MyHatenaBookmarkUsersCount;
18a19
> my $interval = $self->conf->{interval} || 1;
39a41
> sleep $interval if (@permalink);
49c51
< Plagger::Plugin::Filter::HatenaBookmarkUsersCount - Queries Hatena Bookmark users count
---
> Plagger::Plugin::Filter::MyHatenaBookmarkUsersCount - Queries Hatena Bookmark users count
53c55
< - module: Filter::HatenaBookmarkUsersCount
---
> - module: Filter::MyHatenaBookmarkUsersCount
次LivedoorClipUsersCount。intervalの追加とHatenaBookmarkUsersCountみたいに50件ずつ処理するよう修正。
1c1
< package Plagger::Plugin::Filter::LivedoorClipUsersCount;
---
> package Plagger::Plugin::Filter::MyLivedoorClipUsersCount;
18a19
> my $interval = $self->conf->{interval} || 1;
20,38c21,33
< $context->log( info => 'Requesting XMLRPC call to livedoorClip with '
< . scalar(@permalink)
< . ' link(s)' );
<
< my $map = XMLRPC::Lite
< ->proxy('http://rpc.clip.livedoor.com/count')
< ->call( 'clip.getCount', @permalink )
< ->result;
<
< unless ($map) {
< $context->log( warn => 'livedoorClip XMLRPC failed' );
< return;
< }
<
< $context->log( info => 'XMLRPC request success.' );
<
< for my $entry ( $args->{feed}->entries ) {
< if ( defined( my $count = $map->{ $entry->permalink } ) ) {
< $entry->meta->{livedoorclip_users} = $count;
---
> while (my @links = splice(@permalink, 0, 50)) {
> $context->log( info => 'Requesting XMLRPC call to livedoorClip with '
> . scalar(@links)
> . ' link(s)' );
>
> my $map = XMLRPC::Lite
> ->proxy('http://rpc.clip.livedoor.com/count')
> ->call( 'clip.getCount', @links )
> ->result;
>
> unless ($map) {
> $context->log( warn => 'livedoorClip XMLRPC failed' );
> return;
39a35,43
>
> $context->log( info => 'XMLRPC request success.' );
>
> for my $entry ( $args->{feed}->entries ) {
> if ( defined( my $count = $map->{ $entry->permalink } ) ) {
> $entry->meta->{livedoorclip_users} = $count;
> }
> }
> sleep $interval if (@permalink);
最後にDelicious。すでに別物?
とりあえずrssでの取得は複数件の同時取得ができなかったのでjsonに変更。
(json版はすでに差分としてあるけど、15件ずつ取るようにしてAPIも新版を使うようにした)
あと、せっかくなので$entry->meta->{delicious_top_tags}と$entry->meta->{delicious_hash}を追加。
もともとdelicious_rateとdelicious_users取ってるだけなので互換性はあるのかな?
ついでにintervalも追加。
1c1
< package Plagger::Plugin::Filter::Delicious;
---
> package Plagger::Plugin::Filter::MyDelicious;
7c7
< use XML::Feed;
---
> use JSON::Syck;
13c13
< 'update.entry.fixup' => \&update,
---
> 'update.feed.fixup' => \&update,
18c18
< my($self, $context, $args) = @_;
---
> my ( $self, $context, $args ) = @_;
19a20
> my @permalink = map $_->permalink, $args->{feed}->entries;
21,27c22
< sleep $interval;
<
< my $md5 = md5_hex($args->{entry}->permalink);
< my $url = "http://del.icio.us/rss/url/$md5";
<
< $self->log(info => "Going to fetch $url");
<
---
> my $json_url = "http://feeds.delicious.com/feeds/json/url/data?hash=";
29,34d23
< my $res = $ua->fetch($url);
<
< if ($res->is_error) {
< $self->log(error => "Fetch URL $url failed.");
< return;
< }
36c25,30
< my $feed = XML::Feed->parse(\$res->content);
---
> while (my @links = splice(@permalink, 0, 15)) {
> $context->log( info => 'Requesting json call to delicious with '
> . scalar(@links)
> . ' link(s)' );
> my $url = $json_url . (join '&hash=', map { md5_hex($_) } @links);
> $self->log(info => "Going to fetch $url");
38,45c32,36
< unless ($feed) {
< $context->log(warn => "Feed error $url: " . XML::Feed->errstr);
< return;
< }
<
< for my $entry ($feed->entries) {
< my @tag = split / /, ($entry->category || '');
< @tag or next;
---
> my $res = $ua->fetch($url);
> if ($res->is_error) {
> $context->log(warn => "Fetch error $url: " . $res->http_response->message);
> next;
> }
47,48c38,41
< for my $tag (@tag) {
< $args->{entry}->add_tag($tag);
---
> my $data = JSON::Syck::Load($res->content);
> unless (ref $data eq 'ARRAY') {
> $context->log(warn => "json parse error: $data");
> next;
50d42
< }
52,56c44,52
< my $delicious_users = $feed->entries;
< if ($delicious_users >= 30 && $self->conf->{scrape_big_numbers}) {
< my $url = "http://del.icio.us/url/$md5";
< $self->log(info => "users count is more than 30. Trying to scrape from $url.");
< sleep $interval;
---
> $context->log( info => 'json request success.' );
> my %delicious = {};
> for my $h ( @{$data} ) {
> next if !$h || !$h->{url};
> $h->{url} =~ s{\\/}{/}mosixg;
> $delicious{ $h->{url} } = $h;
> }
> for my $entry ($args->{feed}->entries) {
> next if (not defined(my $info = $delicious{$entry->permalink}));
58c54
< my $res = $ua->fetch($url);
---
> my $delicious_users = $info->{total_posts} || 0;
60,62c56,65
< if ($res->is_error) {
< $context->log(warn => "Fetch error $url: " . $res->http_response->message);
< return;
---
> $entry->meta->{delicious_rate} = rate_of_color($delicious_users);
> $entry->meta->{delicious_users} = $delicious_users;
> $entry->meta->{delicious_top_tags} = $info->{top_tags};
> $entry->meta->{delicious_hash} = $info->{hash};
>
> next if (!$info->{top_tags} || !(ref $info->{top_tags} eq 'HASH'));
> for my $tag ( keys %{$info->{top_tags}} ) {
> $entry->add_tag($tag);
> $self->log(debug => "add tag $tag");
> }
64,66c67
<
< $delicious_users =
< ( $res->content =~ m#<h4[^>]*>[^<>]*this url has been saved by\D+(\d+)#s )[0];
---
> sleep $interval if (@permalink);
68,70d68
< $args->{entry}->meta->{delicious_rate} = rate_of_color($delicious_users);
< $args->{entry}->meta->{delicious_users} = $delicious_users;
< $self->log(info => "set delicious_users to $delicious_users");
85c83
< Plagger::Plugin::Filter::Delicious - Fetch tags and users count from del.icio.us
---
> Plagger::Plugin::Filter::MyDelicious - Fetch tags and users count from del.icio.us
1c1
< package Plagger::Plugin::Filter::HatenaBookmarkUsersCount;
---
> package Plagger::Plugin::Filter::MyHatenaBookmarkUsersCount;
18a19
> my $interval = $self->conf->{interval} || 1;
39a41
> sleep $interval if (@permalink);
49c51
< Plagger::Plugin::Filter::HatenaBookmarkUsersCount - Queries Hatena Bookmark users count
---
> Plagger::Plugin::Filter::MyHatenaBookmarkUsersCount - Queries Hatena Bookmark users count
53c55
< - module: Filter::HatenaBookmarkUsersCount
---
> - module: Filter::MyHatenaBookmarkUsersCount
次LivedoorClipUsersCount。intervalの追加とHatenaBookmarkUsersCountみたいに50件ずつ処理するよう修正。
1c1
< package Plagger::Plugin::Filter::LivedoorClipUsersCount;
---
> package Plagger::Plugin::Filter::MyLivedoorClipUsersCount;
18a19
> my $interval = $self->conf->{interval} || 1;
20,38c21,33
< $context->log( info => 'Requesting XMLRPC call to livedoorClip with '
< . scalar(@permalink)
< . ' link(s)' );
<
< my $map = XMLRPC::Lite
< ->proxy('http://rpc.clip.livedoor.com/count')
< ->call( 'clip.getCount', @permalink )
< ->result;
<
< unless ($map) {
< $context->log( warn => 'livedoorClip XMLRPC failed' );
< return;
< }
<
< $context->log( info => 'XMLRPC request success.' );
<
< for my $entry ( $args->{feed}->entries ) {
< if ( defined( my $count = $map->{ $entry->permalink } ) ) {
< $entry->meta->{livedoorclip_users} = $count;
---
> while (my @links = splice(@permalink, 0, 50)) {
> $context->log( info => 'Requesting XMLRPC call to livedoorClip with '
> . scalar(@links)
> . ' link(s)' );
>
> my $map = XMLRPC::Lite
> ->proxy('http://rpc.clip.livedoor.com/count')
> ->call( 'clip.getCount', @links )
> ->result;
>
> unless ($map) {
> $context->log( warn => 'livedoorClip XMLRPC failed' );
> return;
39a35,43
>
> $context->log( info => 'XMLRPC request success.' );
>
> for my $entry ( $args->{feed}->entries ) {
> if ( defined( my $count = $map->{ $entry->permalink } ) ) {
> $entry->meta->{livedoorclip_users} = $count;
> }
> }
> sleep $interval if (@permalink);
最後にDelicious。すでに別物?
とりあえずrssでの取得は複数件の同時取得ができなかったのでjsonに変更。
(json版はすでに差分としてあるけど、15件ずつ取るようにしてAPIも新版を使うようにした)
あと、せっかくなので$entry->meta->{delicious_top_tags}と$entry->meta->{delicious_hash}を追加。
もともとdelicious_rateとdelicious_users取ってるだけなので互換性はあるのかな?
ついでにintervalも追加。
1c1
< package Plagger::Plugin::Filter::Delicious;
---
> package Plagger::Plugin::Filter::MyDelicious;
7c7
< use XML::Feed;
---
> use JSON::Syck;
13c13
< 'update.entry.fixup' => \&update,
---
> 'update.feed.fixup' => \&update,
18c18
< my($self, $context, $args) = @_;
---
> my ( $self, $context, $args ) = @_;
19a20
> my @permalink = map $_->permalink, $args->{feed}->entries;
21,27c22
< sleep $interval;
<
< my $md5 = md5_hex($args->{entry}->permalink);
< my $url = "http://del.icio.us/rss/url/$md5";
<
< $self->log(info => "Going to fetch $url");
<
---
> my $json_url = "http://feeds.delicious.com/feeds/json/url/data?hash=";
29,34d23
< my $res = $ua->fetch($url);
<
< if ($res->is_error) {
< $self->log(error => "Fetch URL $url failed.");
< return;
< }
36c25,30
< my $feed = XML::Feed->parse(\$res->content);
---
> while (my @links = splice(@permalink, 0, 15)) {
> $context->log( info => 'Requesting json call to delicious with '
> . scalar(@links)
> . ' link(s)' );
> my $url = $json_url . (join '&hash=', map { md5_hex($_) } @links);
> $self->log(info => "Going to fetch $url");
38,45c32,36
< unless ($feed) {
< $context->log(warn => "Feed error $url: " . XML::Feed->errstr);
< return;
< }
<
< for my $entry ($feed->entries) {
< my @tag = split / /, ($entry->category || '');
< @tag or next;
---
> my $res = $ua->fetch($url);
> if ($res->is_error) {
> $context->log(warn => "Fetch error $url: " . $res->http_response->message);
> next;
> }
47,48c38,41
< for my $tag (@tag) {
< $args->{entry}->add_tag($tag);
---
> my $data = JSON::Syck::Load($res->content);
> unless (ref $data eq 'ARRAY') {
> $context->log(warn => "json parse error: $data");
> next;
50d42
< }
52,56c44,52
< my $delicious_users = $feed->entries;
< if ($delicious_users >= 30 && $self->conf->{scrape_big_numbers}) {
< my $url = "http://del.icio.us/url/$md5";
< $self->log(info => "users count is more than 30. Trying to scrape from $url.");
< sleep $interval;
---
> $context->log( info => 'json request success.' );
> my %delicious = {};
> for my $h ( @{$data} ) {
> next if !$h || !$h->{url};
> $h->{url} =~ s{\\/}{/}mosixg;
> $delicious{ $h->{url} } = $h;
> }
> for my $entry ($args->{feed}->entries) {
> next if (not defined(my $info = $delicious{$entry->permalink}));
58c54
< my $res = $ua->fetch($url);
---
> my $delicious_users = $info->{total_posts} || 0;
60,62c56,65
< if ($res->is_error) {
< $context->log(warn => "Fetch error $url: " . $res->http_response->message);
< return;
---
> $entry->meta->{delicious_rate} = rate_of_color($delicious_users);
> $entry->meta->{delicious_users} = $delicious_users;
> $entry->meta->{delicious_top_tags} = $info->{top_tags};
> $entry->meta->{delicious_hash} = $info->{hash};
>
> next if (!$info->{top_tags} || !(ref $info->{top_tags} eq 'HASH'));
> for my $tag ( keys %{$info->{top_tags}} ) {
> $entry->add_tag($tag);
> $self->log(debug => "add tag $tag");
> }
64,66c67
<
< $delicious_users =
< ( $res->content =~ m#<h4[^>]*>[^<>]*this url has been saved by\D+(\d+)#s )[0];
---
> sleep $interval if (@permalink);
68,70d68
< $args->{entry}->meta->{delicious_rate} = rate_of_color($delicious_users);
< $args->{entry}->meta->{delicious_users} = $delicious_users;
< $self->log(info => "set delicious_users to $delicious_users");
85c83
< Plagger::Plugin::Filter::Delicious - Fetch tags and users count from del.icio.us
---
> Plagger::Plugin::Filter::MyDelicious - Fetch tags and users count from del.icio.us
diffはunified(diff -u形式)で書いてもらえると読みやすいかも。