diff -u Filter_org/Delicious.pm Filter/Delicious.pm
--- Filter_org/Delicious.pm 2006-12-05 16:01:24.000000000 +0900
+++ Filter/Delicious.pm 2008-09-10 16:23:59.000000000 +0900
@@ -4,70 +4,68 @@
use Digest::MD5 qw(md5_hex);
use Plagger::UserAgent;
-use XML::Feed;
+use JSON::Syck;
sub register {
my($self, $context) = @_;
$context->register_hook(
$self,
- 'update.entry.fixup' => \&update,
+ 'update.feed.fixup' => \&update,
);
}
sub update {
my($self, $context, $args) = @_;
+ my @permalink = map $_->permalink, $args->{feed}->entries;
my $interval = $self->conf->{interval} || 1;
- sleep $interval;
-
- my $md5 = md5_hex($args->{entry}->permalink);
- my $url = "http://del.icio.us/rss/url/$md5";
-
- $self->log(info => "Going to fetch $url");
-
+ my $json_url = "http://feeds.delicious.com/feeds/json/url/data?hash=";
my $ua = Plagger::UserAgent->new;
- my $res = $ua->fetch($url);
-
- if ($res->is_error) {
- $self->log(error => "Fetch URL $url failed.");
- return;
- }
- my $feed = XML::Feed->parse(\$res->content);
+ my %delicious = {};
+ while (my @links = splice(@permalink, 0, 15)) {
+ $context->log( info => 'Requesting json call to delicious with '
+ . scalar(@links)
+ . ' link(s)' );
+ my $url = $json_url . (join '&hash=', map { md5_hex($_) } @links);
+ $self->log(info => "Going to fetch $url");
- unless ($feed) {
- $context->log(warn => "Feed error $url: " . XML::Feed->errstr);
- return;
- }
+ my $res = $ua->fetch($url);
+ if ($res->is_error) {
+ $context->log(warn => "Fetch error $url: " . $res->http_response->message);
+ next;
+ }
- for my $entry ($feed->entries) {
- my @tag = split / /, ($entry->category || '');
- @tag or next;
+ my $data = JSON::Syck::Load($res->content);
+ unless (ref $data eq 'ARRAY') {
+ $context->log(warn => "json parse error: $data");
+ next;
+ }
- for my $tag (@tag) {
- $args->{entry}->add_tag($tag);
+ $context->log( info => 'json request success.' );
+ for my $h ( @{$data} ) {
+ next if !$h || !$h->{url};
+ $h->{url} =~ s{\\/}{/}mosixg;
+ $delicious{ $h->{url} } = $h;
}
+ sleep $interval if (@permalink);
}
+ for my $entry ($args->{feed}->entries) {
+ next if (not defined(my $info = $delicious{$entry->permalink}));
- my $delicious_users = $feed->entries;
- if ($delicious_users >= 30 && $self->conf->{scrape_big_numbers}) {
- my $url = "http://del.icio.us/url/$md5";
- $self->log(info => "users count is more than 30. Trying to scrape from $url.");
- sleep $interval;
-
- my $res = $ua->fetch($url);
+ my $delicious_users = $info->{total_posts} || 0;
- if ($res->is_error) {
- $context->log(warn => "Fetch error $url: " . $res->http_response->message);
- return;
+ $entry->meta->{delicious_rate} = rate_of_color($delicious_users);
+ $entry->meta->{delicious_users} = $delicious_users;
+ $entry->meta->{delicious_top_tags} = $info->{top_tags};
+ $entry->meta->{delicious_hash} = $info->{hash};
+
+ next if (!$info->{top_tags} || !(ref $info->{top_tags} eq 'HASH'));
+ for my $tag ( keys %{$info->{top_tags}} ) {
+ $entry->add_tag($tag);
+ $self->log(debug => "add tag $tag");
}
-
- $delicious_users =
- ( $res->content =~ m#<h4[^>]*>[^<>]*this url has been saved by\D+(\d+)#s )[0];
}
- $args->{entry}->meta->{delicious_rate} = rate_of_color($delicious_users);
- $args->{entry}->meta->{delicious_users} = $delicious_users;
- $self->log(info => "set delicious_users to $delicious_users");
}
sub rate_of_color {
@@ -106,6 +104,8 @@
=head1 SEE ALSO
-L<Plagger>, L<http://del.icio.us/help/>
+L<Plagger>, L<http://delicious.com/help/api>
+L<Plagger>, L<http://delicious.com/help/feeds/>
+L<Plagger>, L<http://delicious.com/help/html>
=cut
diff -u Filter_org/HatenaBookmarkUsersCount.pm Filter/HatenaBookmarkUsersCount.pm
--- Filter_org/HatenaBookmarkUsersCount.pm 2006-12-05 16:01:26.000000000 +0900
+++ Filter/HatenaBookmarkUsersCount.pm 2008-09-10 16:16:13.000000000 +0900
@@ -16,6 +16,7 @@
my($self, $context, $args) = @_;
my @permalink = map $_->permalink, $args->{feed}->entries;
+ my $interval = $self->conf->{interval} || 1;
while (my @links = splice(@permalink, 0, 50)) {
$context->log(info => 'Requesting XMLRPC call to Hatena Bookmark with ' . scalar(@links) . ' link(s)');
@@ -37,6 +38,7 @@
$entry->meta->{hatenabookmark_users} = $count;
}
}
+ sleep $interval if (@permalink);
}
}
diff -u Filter_org/LivedoorClipUsersCount.pm Filter/LivedoorClipUsersCount.pm
--- Filter_org/LivedoorClipUsersCount.pm 2006-12-05 16:01:27.000000000 +0900
+++ Filter/LivedoorClipUsersCount.pm 2008-09-10 16:14:24.000000000 +0900
@@ -16,27 +16,31 @@
my ( $self, $context, $args ) = @_;
my @permalink = map $_->permalink, $args->{feed}->entries;
+ my $interval = $self->conf->{interval} || 1;
- $context->log( info => 'Requesting XMLRPC call to livedoorClip with '
- . scalar(@permalink)
- . ' link(s)' );
-
- my $map = XMLRPC::Lite
- ->proxy('http://rpc.clip.livedoor.com/count')
- ->call( 'clip.getCount', @permalink )
- ->result;
-
- unless ($map) {
- $context->log( warn => 'livedoorClip XMLRPC failed' );
- return;
- }
+ while (my @links = splice(@permalink, 0, 50)) {
+ $context->log( info => 'Requesting XMLRPC call to livedoorClip with '
+ . scalar(@links)
+ . ' link(s)' );
+
+ my $map = XMLRPC::Lite
+ ->proxy('http://rpc.clip.livedoor.com/count')
+ ->call( 'clip.getCount', @links )
+ ->result;
+
+ unless ($map) {
+ $context->log( warn => 'livedoorClip XMLRPC failed' );
+ return;
+ }
- $context->log( info => 'XMLRPC request success.' );
+ $context->log( info => 'XMLRPC request success.' );
- for my $entry ( $args->{feed}->entries ) {
- if ( defined( my $count = $map->{ $entry->permalink } ) ) {
- $entry->meta->{livedoorclip_users} = $count;
+ for my $entry ( $args->{feed}->entries ) {
+ if ( defined( my $count = $map->{ $entry->permalink } ) ) {
+ $entry->meta->{livedoorclip_users} = $count;
+ }
}
+ sleep $interval if (@permalink);
}
}
--- Filter_org/Delicious.pm 2006-12-05 16:01:24.000000000 +0900
+++ Filter/Delicious.pm 2008-09-10 16:23:59.000000000 +0900
@@ -4,70 +4,68 @@
use Digest::MD5 qw(md5_hex);
use Plagger::UserAgent;
-use XML::Feed;
+use JSON::Syck;
sub register {
my($self, $context) = @_;
$context->register_hook(
$self,
- 'update.entry.fixup' => \&update,
+ 'update.feed.fixup' => \&update,
);
}
sub update {
my($self, $context, $args) = @_;
+ my @permalink = map $_->permalink, $args->{feed}->entries;
my $interval = $self->conf->{interval} || 1;
- sleep $interval;
-
- my $md5 = md5_hex($args->{entry}->permalink);
- my $url = "http://del.icio.us/rss/url/$md5";
-
- $self->log(info => "Going to fetch $url");
-
+ my $json_url = "http://feeds.delicious.com/feeds/json/url/data?hash=";
my $ua = Plagger::UserAgent->new;
- my $res = $ua->fetch($url);
-
- if ($res->is_error) {
- $self->log(error => "Fetch URL $url failed.");
- return;
- }
- my $feed = XML::Feed->parse(\$res->content);
+ my %delicious = {};
+ while (my @links = splice(@permalink, 0, 15)) {
+ $context->log( info => 'Requesting json call to delicious with '
+ . scalar(@links)
+ . ' link(s)' );
+ my $url = $json_url . (join '&hash=', map { md5_hex($_) } @links);
+ $self->log(info => "Going to fetch $url");
- unless ($feed) {
- $context->log(warn => "Feed error $url: " . XML::Feed->errstr);
- return;
- }
+ my $res = $ua->fetch($url);
+ if ($res->is_error) {
+ $context->log(warn => "Fetch error $url: " . $res->http_response->message);
+ next;
+ }
- for my $entry ($feed->entries) {
- my @tag = split / /, ($entry->category || '');
- @tag or next;
+ my $data = JSON::Syck::Load($res->content);
+ unless (ref $data eq 'ARRAY') {
+ $context->log(warn => "json parse error: $data");
+ next;
+ }
- for my $tag (@tag) {
- $args->{entry}->add_tag($tag);
+ $context->log( info => 'json request success.' );
+ for my $h ( @{$data} ) {
+ next if !$h || !$h->{url};
+ $h->{url} =~ s{\\/}{/}mosixg;
+ $delicious{ $h->{url} } = $h;
}
+ sleep $interval if (@permalink);
}
+ for my $entry ($args->{feed}->entries) {
+ next if (not defined(my $info = $delicious{$entry->permalink}));
- my $delicious_users = $feed->entries;
- if ($delicious_users >= 30 && $self->conf->{scrape_big_numbers}) {
- my $url = "http://del.icio.us/url/$md5";
- $self->log(info => "users count is more than 30. Trying to scrape from $url.");
- sleep $interval;
-
- my $res = $ua->fetch($url);
+ my $delicious_users = $info->{total_posts} || 0;
- if ($res->is_error) {
- $context->log(warn => "Fetch error $url: " . $res->http_response->message);
- return;
+ $entry->meta->{delicious_rate} = rate_of_color($delicious_users);
+ $entry->meta->{delicious_users} = $delicious_users;
+ $entry->meta->{delicious_top_tags} = $info->{top_tags};
+ $entry->meta->{delicious_hash} = $info->{hash};
+
+ next if (!$info->{top_tags} || !(ref $info->{top_tags} eq 'HASH'));
+ for my $tag ( keys %{$info->{top_tags}} ) {
+ $entry->add_tag($tag);
+ $self->log(debug => "add tag $tag");
}
-
- $delicious_users =
- ( $res->content =~ m#<h4[^>]*>[^<>]*this url has been saved by\D+(\d+)#s )[0];
}
- $args->{entry}->meta->{delicious_rate} = rate_of_color($delicious_users);
- $args->{entry}->meta->{delicious_users} = $delicious_users;
- $self->log(info => "set delicious_users to $delicious_users");
}
sub rate_of_color {
@@ -106,6 +104,8 @@
=head1 SEE ALSO
-L<Plagger>, L<http://del.icio.us/help/>
+L<Plagger>, L<http://delicious.com/help/api>
+L<Plagger>, L<http://delicious.com/help/feeds/>
+L<Plagger>, L<http://delicious.com/help/html>
=cut
diff -u Filter_org/HatenaBookmarkUsersCount.pm Filter/HatenaBookmarkUsersCount.pm
--- Filter_org/HatenaBookmarkUsersCount.pm 2006-12-05 16:01:26.000000000 +0900
+++ Filter/HatenaBookmarkUsersCount.pm 2008-09-10 16:16:13.000000000 +0900
@@ -16,6 +16,7 @@
my($self, $context, $args) = @_;
my @permalink = map $_->permalink, $args->{feed}->entries;
+ my $interval = $self->conf->{interval} || 1;
while (my @links = splice(@permalink, 0, 50)) {
$context->log(info => 'Requesting XMLRPC call to Hatena Bookmark with ' . scalar(@links) . ' link(s)');
@@ -37,6 +38,7 @@
$entry->meta->{hatenabookmark_users} = $count;
}
}
+ sleep $interval if (@permalink);
}
}
diff -u Filter_org/LivedoorClipUsersCount.pm Filter/LivedoorClipUsersCount.pm
--- Filter_org/LivedoorClipUsersCount.pm 2006-12-05 16:01:27.000000000 +0900
+++ Filter/LivedoorClipUsersCount.pm 2008-09-10 16:14:24.000000000 +0900
@@ -16,27 +16,31 @@
my ( $self, $context, $args ) = @_;
my @permalink = map $_->permalink, $args->{feed}->entries;
+ my $interval = $self->conf->{interval} || 1;
- $context->log( info => 'Requesting XMLRPC call to livedoorClip with '
- . scalar(@permalink)
- . ' link(s)' );
-
- my $map = XMLRPC::Lite
- ->proxy('http://rpc.clip.livedoor.com/count')
- ->call( 'clip.getCount', @permalink )
- ->result;
-
- unless ($map) {
- $context->log( warn => 'livedoorClip XMLRPC failed' );
- return;
- }
+ while (my @links = splice(@permalink, 0, 50)) {
+ $context->log( info => 'Requesting XMLRPC call to livedoorClip with '
+ . scalar(@links)
+ . ' link(s)' );
+
+ my $map = XMLRPC::Lite
+ ->proxy('http://rpc.clip.livedoor.com/count')
+ ->call( 'clip.getCount', @links )
+ ->result;
+
+ unless ($map) {
+ $context->log( warn => 'livedoorClip XMLRPC failed' );
+ return;
+ }
- $context->log( info => 'XMLRPC request success.' );
+ $context->log( info => 'XMLRPC request success.' );
- for my $entry ( $args->{feed}->entries ) {
- if ( defined( my $count = $map->{ $entry->permalink } ) ) {
- $entry->meta->{livedoorclip_users} = $count;
+ for my $entry ( $args->{feed}->entries ) {
+ if ( defined( my $count = $map->{ $entry->permalink } ) ) {
+ $entry->meta->{livedoorclip_users} = $count;
+ }
}
+ sleep $interval if (@permalink);
}
}
とりいそぎAPIのドメイン変更分だけpatchコミットしました。
http://plagger.org/trac/changeset/2056
ありがとうございます。
近くにいるコミット可能な人に聞いたら「テスト無しのパッチなんてパッチ無しのテストだけよりひどい」といわれたのでテスト書かなきゃなーと思いつつ放置してます。