daniell/pcurse
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Wiki Activity

Using threads, because it's fun:)

Browse Source
This commit is contained in:
Daniel Lysfjord
2019-07-07 00:57:07 +02:00
parent f3a40e3b50
commit b087c00991
2 changed files with 219 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
pcurse
View File

@@ -1,44 +1,114 @@
#!/usr/bin/env perl #!/usr/bin/env perl
use strict; use strict;
use warnings; no warnings 'all';
use lib '.'; use lib '.';
use pcurse; use pcurse;
use feature ':5.10';
select(STDOUT); select(STDOUT);
$| = 1; $| = 1;
my $ret; my $ret;
my $msg; my $msg;
my %opts = pcurse::parse_arguments; my @toupd;
$opts{'config'} = $ENV{'HOME'}.'/.pcurse/config.json' unless($opts{'config'}); my %jobs;
my $conf = pcurse::load_config($opts{'config'}); my $workers;
my $opts = pcurse::parse_arguments;
$opts->{'config'} = $ENV{'HOME'}.'/.pcurse/config.json' unless(defined($opts->{'config'}));
my $conf = pcurse::load_config($opts->{'config'});
$conf = pcurse::check_config($conf); $conf = pcurse::check_config($conf);
($ret,$msg) = pcurse::save_config($opts{'config'},$conf); ($ret,$msg) = pcurse::save_config($conf->{'config'},$conf);
print $msg."\n" unless($ret); unless($ret) {
print $msg."\n";
exit 0;
}
$conf = pcurse::merge_opts($opts,$conf);
my $addons = pcurse::load_addons($conf->{'addons'}); my $addons = pcurse::load_addons($conf->{'addons'});
say 'Loaded '.scalar(@{$addons}).' addons';
my $pool = pcurse::init_pool($conf->{'workers'});
my $worki = 0;
print 'Checking for updates ';
foreach my $addon(@{$addons}) { foreach my $addon(@{$addons}) {
print 'Found '.$addon->{'name'}.'('.$addon->{'version'}.')'; my $workid = $pool->job('check',$addon,$conf);
if(exists($addon->{'uri'})) { $jobs{$workid}{'todo'} = 'check';
my $html = pcurse::html_get($conf->{'baseuri'}.$addon->{'uri'}); $jobs{$workid}{'job'} = $addon;
my $fileid = pcurse::get_latest_file_id($html,$addon->{'uri'}); $jobs{$workid}{'i'} = $worki;
if($fileid) { $worki++;
my $version = pcurse::get_product_version($html,$addon->{'uri'},$fileid); }
print ' - latest version is '.$version; while(scalar(keys %jobs)) {
if($version && ($version ne $addon->{'version'})) { my $jobid;
unless($opts{'test'}) { if(my @jres = $pool->result_any(\$jobid)) {
print ' - updating'; if($jres[0] == 1) {
my $ret = pcurse::update($conf->{'baseuri'}.$addon->{'uri'},$fileid,$conf->{'wowpath'}); my $a = $jres[1]; #Addon data structure
if($ret) { my $i = $jobs{$jobid}{'i'}; #ID from @{$addons}
print ' - done'; my $v = $jres[1]->{'targetversion'}; #Version we're trying to fetch
$addon->{'version'} = $version; die "Not enough params: a: $a, i: $i, v: $v" unless(defined($a) && defined($i) && defined($v));
} else { my $nextjob = [ $i, $a, $v ];
print ' - failed'; push(@toupd,$nextjob);
} } elsif($jres[1] eq 'No need to update') {
} } else {
say 'Error during check: '.$jres[1];
}
delete $jobs{$jobid};
print '.';
}
}
print "\n";
print 'Downloading updates ' if(scalar(@toupd));
foreach my $a(@toupd) {
my $id = $a->[0];
my $addon = $a->[1];
my $version = $a->[2];
my $workid = $pool->job('download',$conf->{'baseuri'}.$addon->{'uri'},$addon->{'fileid'});
$jobs{$workid}{'todo'} = 'download';
$jobs{$workid}{'job'} = [ $conf->{'baseuri'}.$addon->{'uri'},$addon->{'fileid'} ];
$jobs{$workid}{'id'} = $id;
$jobs{$workid}{'tv'} = $version;
}
my %tounpack;
while(scalar(keys %jobs)) {
my $jobid;
if(my @jres = $pool->result_any(\$jobid)) {
if($jres[0]{'retval'} == 1) {
my $ret = $jres[0]{'retval'};
my $filename = $jres[0]{'filename'};
my $file = $jres[0]{'filecontent'};
my $version = $jobs{$jobid}{'tv'};
if(defined($filename)) {
#say 'Going to unpack file '.$filename.' containing version '.$version;
$tounpack{$filename} = [ $jobs{$jobid}{'id'},$file,$version ];
} else {
#print Dumper @jres;
my $ai = $jobs{$jobid}{'id'};
my $an = $addons->[$ai]->{'name'};
say 'Passed an empty filename? in update';
} }
} else { } else {
print 'Could not find file id for '.$addon->{'name'}; my $uri = shift;
say 'Download failed for '.$uri;
} }
delete $jobs{$jobid};
print '.';
} }
print "\n";
} }
$pool->shutdown;
print "\n" if(scalar(@toupd));
say 'Unpacking updates ' if(scalar(keys %tounpack));
foreach my $unpacking(keys %tounpack) {
my $id = $tounpack{$unpacking}->[0];
my $file = $tounpack{$unpacking}->[1];
my $version = $tounpack{$unpacking}->[2];
if(pcurse::update($unpacking,$file,$conf->{'wowpath'})) {
say 'Updated '.$addons->[$id]->{'name'}.' from version '.$addons->[$id]->{'version'}.' to '.$version;
$addons->[$id]->{'version'} = $version;
} else {
say 'Unpacking failed for '.$unpacking;
}
}
($ret,$msg) = pcurse::save_config($conf->{'addons'},$addons); ($ret,$msg) = pcurse::save_config($conf->{'addons'},$addons);
print $msg."\n" unless($ret); print $msg."\n" unless($ret);

161
pcurse.pm
View File

@@ -1,21 +1,44 @@
#!/usr/bin/env perl #!/usr/bin/env perl
package pcurse; package pcurse;
use JSON; use strict;
use HTML::HTML5::Parser; no warnings 'all';
use LWP::UserAgent ();
use Getopt::Long; use Getopt::Long;
use Archive::Extract; use Archive::Extract;
use Thread::Pool;
use JSON;
use LWP::UserAgent;
use HTML::HTML5::Parser;
use IO::Socket::SSL;
use feature ':5.10';
sub merge_opts {
my $opts = shift;
my $conf = shift;
if(defined($opts->{'debug'})) {
print 'We got opts:'."\n";
print Dumper $opts;
print 'We got conf:'."\n";
print Dumper $conf;
}
foreach my $k(keys %{$opts}) {
next unless(defined($opts->{$k}));
$conf->{$k} = $opts->{$k};
}
return $conf;
}
sub parse_arguments { sub parse_arguments {
my %toret; my $toret;
GetOptions ( Getopt::Long::GetOptions (
"verbose+" => \$toret{'verbose'}, "verbose" => \$toret->{'verbose'},
"wowpath=s" => \$toret{'wowpath'}, "wowpath=s" => \$toret->{'wowpath'},
"baseuri=s" => \$toret{'baseuri'}, "baseuri=s" => \$toret->{'baseuri'},
"config=s" => \$toret{'config'}, "config=s" => \$toret->{'config'},
"test" => \$toret{'test'}, "test" => \$toret->{'test'},
"workers" => \$toret->{'workers'},
"debug" => \$toret->{'debug'},
); );
return %toret; return $toret;
} }
sub load_config { sub load_config {
@@ -55,8 +78,9 @@ sub check_config {
sub sane_defaults { sub sane_defaults {
my $in = shift; my $in = shift;
$in->{'baseuri'} = 'https://www.curseforge.com' unless(exists($in->{'baseuri'})); $in->{'baseuri'} = 'https://www.curseforge.com' unless(exists($in->{'baseuri'}));
$in->{'config'} = $ENV{'HOME'}.'/.pcurse/config.json' unless(exists($in->{'config'})); $in->{'baseuri'} =~ s/^http/https/ unless($in->{'baseuri'} =~ m/^https/);
$in->{'addons'} = $ENV{'HOME'}.'/.pcurse/addons.json' unless(exists($in->{'addons'})); $in->{'addons'} = $ENV{'HOME'}.'/.pcurse/addons.json' unless(exists($in->{'addons'}));
$in->{'workers'} = "4" unless(exists($in->{'workers'}));
return $in; return $in;
} }
@@ -93,13 +117,21 @@ sub load_addons {
sub save_config { sub save_config {
my $json = JSON->new; my $json = JSON->new;
$json->convert_blessed;
$json->allow_nonref;
$json->allow_tags;
$json->allow_blessed;
my $file = shift; my $file = shift;
my $json_data = shift; my $json_data = shift;
my $text = $json->pretty->encode($json_data); if(ref $json_data eq 'ARRAY' or ref $json_data eq 'HASH') {
open my $fh, ">", $file or return (0,'Could not open '.$file.' for writing: '.$!); my $text = $json->pretty->encode($json_data);
print $fh $text; open my $fh, ">", $file or return (0,'Could not open '.$file.' for writing: '.$!);
close $fh; print $fh $text;
return (1,$file.' saved successfully'); close $fh;
return (1,$file.' saved successfully');
} else {
say 'Invalid format on passed data (not a HASH or ARRAY ref)';
}
} }
sub import_json { sub import_json {
@@ -183,34 +215,44 @@ sub find_in_html {
} else { } else {
#This means we're on our own = whatever we're getting here is a html document as a string, unparsed. #This means we're on our own = whatever we're getting here is a html document as a string, unparsed.
my $parser = HTML::HTML5::Parser->new(); my $parser = HTML::HTML5::Parser->new();
my @file = split(/\n/, $html); if(defined($html)) {
foreach my $line(@file) { my @file = split(/\n/, $html);
if($line =~ m/$sstring/) { foreach my $line(@file) {
my $parsed = $parser->parse_balanced_chunk($line); if($line =~ m/$sstring/) {
my @nodes = $parsed->nonBlankChildNodes(); my $parsed = $parser->parse_balanced_chunk($line);
foreach my $node(@nodes) { my @nodes = $parsed->nonBlankChildNodes();
my @atr = $node->attributes(); foreach my $node(@nodes) {
if($mode eq 'dlstring') { my @atr = $node->attributes();
my $href = $node->getAttribute('href'); if($mode eq 'dlstring') {
$retstr = (split(/$sstring/, $href,2))[1]; my $href = $node->getAttribute('href');
} elsif($mode eq 'vstring') { $retstr = (split(/$sstring/, $href,2))[1];
$retstr = $node->getAttribute('data-name'); } elsif($mode eq 'vstring') {
$retstr = $node->getAttribute('data-name');
}
return $retstr if($retstr);
} }
return $retstr if($retstr);
} }
} }
} else {
return undef;
} }
} }
return undef; return undef;
} }
sub download_update {
my $uri = shift;
my $fileid = shift;
$uri .= '/download/'.$fileid.'/file';
my ($ret,$filename,$file) = pcurse::download($uri);
return (1,$filename,$file) if($ret);
return (0,undef,undef);
}
sub update { sub update {
my $uri = shift; my $filename = shift;
my $fileid = shift; my $file = shift;
my $targetpath = shift; my $targetpath = shift;
$uri .= '/download/'.$fileid.'/file';
my ($filename,$file) = pcurse::download($uri);
unless(-e "/tmp/$filename") { unless(-e "/tmp/$filename") {
open my $fh, '>', "/tmp/$filename" or return 0; open my $fh, '>', "/tmp/$filename" or return 0;
print $fh $file; print $fh $file;
@@ -226,9 +268,52 @@ sub update {
sub download { sub download {
my $uri = shift; my $uri = shift;
my $file = pcurse::http_get($uri); my $file = pcurse::http_get($uri);
my $filename = $file->filename; if(defined($file)) {
my $content = $file->decoded_content; my $filename = $file->filename;
return ($filename,$content); my $content = $file->decoded_content;
return (1,$filename,$content);
} else {
return (0,$uri,undef);
}
}
sub init_pool {
my $w = shift;
my $p = Thread::Pool->new( {
workers => $w,
do => sub {
my $todo = shift;
if($todo eq 'check') {
my $addon = shift;
my $conf = shift;
my $html = pcurse::html_get($conf->{'baseuri'}.$addon->{'uri'});
my $fileid = pcurse::get_latest_file_id($html,$addon->{'uri'});
if($fileid) {
$addon->{'fileid'} = $fileid;
my $version = pcurse::get_product_version($html,$addon->{'uri'},$fileid);
if($version && ($version ne $addon->{'version'})) {
unless($conf->{'test'}) {
$addon->{'targetversion'} = $version;
return (1,$addon);
}
} else {
return (0,'No need to update');
}
} else {
return (0,'Could not find file id for '.$addon->{'name'});
}
} elsif($todo eq 'download') {
my $uri = shift;
my $fileid = shift;
my ($ret,$filename,$file) = pcurse::download_update($uri,$fileid);
return { retval => $ret, filename => $filename, filecontent => $file } if($ret);
return { retval => 0, filename => undef, filecontent => undef };
} else {
return (0,'Unknown task');
}
},
});
return $p;
} }
1; 1;