Skip to content

Commit d9f5c77

Browse files
toddbrunertoddbruner
committed
4.3 Release
1 parent 64ceb80 commit d9f5c77

31 files changed

+1343
-249
lines changed

‎etc/test.sqlite.sql‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS regex (
1818
regex_id INTEGER PRIMARY KEY AUTOINCREMENT,
1919
created TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
2020
updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
21+
active BOOLEAN DEFAULT true,
2122
name TEXT NOT NULL,
2223
description TEXT NOT NULL,
2324
match TEXT NOT NULL UNIQUE,
@@ -38,19 +39,22 @@ insert into regex values (
3839
NULL,
3940
NULL,
4041
NULL,
42+
TRUE,
4143
'new closing dispo',
42-
'User Defined Entity', 'new\ closing\ dispo', 'test_entity', 'udef', 101, 1);
44+
'User Defined Entity', 'new closing dispo', 'test_entity', 'udef', 101, 1);
4345
insert into regex values (
4446
NULL,
4547
NULL,
4648
NULL,
49+
TRUE,
4750
'foo',
4851
'User Defined Entity', 'fufoo', 'test_entity', 'udef', 100, 0);
4952

5053
insert into regex values (
5154
NULL,
5255
NULL,
5356
NULL,
57+
TRUE,
5458
'sydney rox',
5559
'User Defined Entity', 'sydney rox', 'test_entity', 'udef', 102, 0);
5660

@@ -81,7 +85,7 @@ CREATE TABLE IF NOT EXISTS apikeys (
8185
apikey_id INTEGER PRIMARY KEY AUTOINCREMENT,
8286
updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
8387
username TEXT NOT NULL,
84-
key TEXT NOT NULL,
88+
apikey TEXT NOT NULL,
8589
lastaccess TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
8690
flairjob BOOLEAN, -- create a flair job and query and read results
8791
regex_ro BOOLEAN, -- read only regexes
@@ -118,21 +122,21 @@ BEGIN
118122
UPDATE admins SET updated=CURRENT_TIMESTAMP WHERE admin_id=NEW.admin_id;
119123
END;
120124

121-
-- CREATE TABLE IF NOT EXISTS jobs (
122-
-- job_id INTEGER PRIMARY KEY AUTOINCREMENT,
123-
-- updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
124-
-- duration DECIMAL(6,4) DEFAULT 0, imgduration DECIMAL(6,4) DEFAULT 0,
125-
-- sourcelen INT DEFAULT 0,
126-
-- images INT DEFAULT 0,
127-
-- entities INT DEFAULT 0
128-
-- );
129-
--
130-
-- CREATE TRIGGER [update_jobs_updated]
131-
-- AFTER UPDATE ON jobs FOR EACH ROW
132-
-- WHEN OLD.updated = NEW.updated OR OLD.updated IS NULL
133-
-- BEGIN
134-
-- UPDATE jobs SET updated=CURRENT_TIMESTAMP WHERE job_id=NEW.job_id;
135-
-- END;
125+
CREATE TABLE IF NOT EXISTS jobs (
126+
job_id INTEGER PRIMARY KEY AUTOINCREMENT,
127+
updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
128+
duration DECIMAL(6,4) DEFAULT 0, imgduration DECIMAL(6,4) DEFAULT 0,
129+
sourcelen INT DEFAULT 0,
130+
images INT DEFAULT 0,
131+
entities INT DEFAULT 0
132+
);
133+
134+
CREATE TRIGGER [update_jobs_updated]
135+
AFTER UPDATE ON jobs FOR EACH ROW
136+
WHEN OLD.updated = NEW.updated OR OLD.updated IS NULL
137+
BEGIN
138+
UPDATE jobs SET updated=CURRENT_TIMESTAMP WHERE job_id=NEW.job_id;
139+
END;
136140

137141
--1 down
138142
DROP TABLE IF EXISTS jobs;

‎etc/testlog.conf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
log4perl.category.Flair = TRACE, FlairLog
1+
log4perl.category.Flair = DEBUG, FlairLog
22
log4perl.appender.FlairLog = Log::Log4perl::Appender::File
33
log4perl.appender.FlairLog.mode = append
44
log4perl.appender.FlairLog.filename = /var/log/flair/test.log
55
log4perl.appender.FlairLog.layout = Log::Log4perl::Layout::PatternLayout
6-
log4perl.appender.FlairLog.layout.ConversionPattern = %d %7p [%P] %15F{1}: %4L %m%n
6+
log4perl.appender.FlairLog.layout.ConversionPattern = %d %7p %10F{1}:%4L %m%n
77

‎lib/Flair.pm‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,15 @@ sub startup ($self) {
7373
);
7474
$processor->do_flairing($job, @args);
7575
});
76+
77+
$self->minion->add_task('flair_bulk' => sub ($job, @args) {
78+
$self->log->debug("Inside flair_bulk");
79+
my $processor = Flair::Processor->new(
80+
log => $self->log, db => $db, scotapi => $scotapi,
81+
config => $self->config, minion => $self->minion,
82+
);
83+
$processor->do_bulk($job, @args);
84+
});
7685

7786
# this is most likely behind a reverse proxy especiallly in a kubernetes/docker env
7887
# so set the env var MOJO_REVERSE_PROXY to the path prefeix (likely /scot-flair)
@@ -173,7 +182,7 @@ sub log_server_startup ($self) {
173182
"| MOJO : ".$Mojolicious::VERSION,
174183
"| Mode : ".$self->mode,
175184
"| DB : ".$self->config->{database}->{uri},
176-
"| INC : ".$inc,
185+
# "| INC : ".$inc,
177186
# "| MOJO_REVERSE_PROXY : ".$ENV{MOJO_REVERSE_PROXY},
178187
"| CONFIG: ", $cfg,
179188
"==========================================================",
@@ -187,10 +196,21 @@ sub catch_error_setup ($self) {
187196
# in eval, don't log, catch later
188197
return;
189198
}
190-
$Log::Log4perl::caller_depth++;
199+
local $Log::Log4perl::caller_depth = $Log::Log4perl::caller_depth+1;
191200
$self->log->error(longmess());
192201
$self->log->fatal(@_);
193202
die @_;
194203
};
204+
#$SIG{'__WARN__'} = sub {
205+
# if ( $^S ){
206+
# # in eval, don't log, catch later
207+
# return;
208+
# }
209+
# my $depth = $Log::Log4perl::caller_depth;
210+
# $Log::Log4perl::caller_depth++;
211+
# $self->log->warn(longmess());
212+
# $self->log->warn(@_);
213+
# $Log::Log4perl::caller_depth = $depth;
214+
#};
195215
}
196216
1;

‎lib/Flair/Config.pm‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,14 @@ sub build_config () {
6161
set_default('S4FLAIR_SCOT_API_KEY', 'xxxxxx');
6262
# uri base
6363
set_default('S4FLAIR_SCOT_API_URI_ROOT', 'https://scot4/api/v1');
64+
set_default('S4FLAIR_SCOT_EXTERNAL_HOSTNAME', 'scot4');
6465

6566
my $logfile = join('/', $ENV{S4FLAIR_LOG_DIR}, $ENV{S4FLAIR_LOG_FILE});
6667

6768
my $config = {
6869
version => $ENV{S4FLAIR_VERSION},
6970
mode => $ENV{S4FLAIR_MODE},
71+
scot_external_hostname => $ENV{S4FLAIR_SCOT_EXTERNAL_HOSTNAME},
7072
install => {
7173
dbfile => $ENV{S4FLAIR_DB_FILE},
7274
admin_user => $ENV{S4FLAIR_ADMIN_USER},
@@ -87,7 +89,7 @@ log4perl.appender.FlairLog = Log::Log4perl::Appender::File
8789
log4perl.appender.FlairLog.mode = append
8890
log4perl.appender.FlairLog.filename = $logfile
8991
log4perl.appender.FlairLog.layout = Log::Log4perl::Layout::PatternLayout
90-
log4perl.appender.FlairLog.layout.ConversionPattern = %d %7p [%P] %15F{1}: %4L %m%n
92+
log4perl.appender.FlairLog.layout.ConversionPattern = %d %5p [%12F:%4L] %m%n
9193
},
9294
},
9395
hypnotoad => {
@@ -130,6 +132,7 @@ log4perl.appender.FlairLog.layout.ConversionPattern = %d %7p [%P] %15F{1}: %4L %
130132
api_key => $ENV{S4FLAIR_SCOT_API_KEY},
131133
uri_root => $ENV{S4FLAIR_SCOT_API_URI_ROOT},
132134
},
135+
flair_job_test => $ENV{S4FLAIR_JOB_TEST} // undef,
133136
};
134137
return $config;
135138
}

‎lib/Flair/Controller/Auth.pm‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -98,10 +98,18 @@ sub valid_header ($self) {
9898
}
9999

100100
sub validate_header ($self, $type, $value) {
101-
return $self->validate_basic($value) if ( $type eq "basic" );
102-
$self->log->debug("not basic");
103-
return $self->validate_apikey($value) if ( $type eq "apikey" );
104-
$self->log->debug("not apikey");
101+
if ($type eq "basic") {
102+
if ($self->validate_basic($value)) {
103+
$self->log->debug("User validated by basic");
104+
return 1;
105+
}
106+
}
107+
if ($type eq "apikey") {
108+
if ($self->validate_apikey($value)) {
109+
$self->log->debug("user validated by authkey");
110+
return 1;
111+
}
112+
}
105113
return undef;
106114
}
107115

@@ -168,9 +176,10 @@ sub log_request ($self, $user=undef) {
168176
my @jlines = map { " "x20 . qq|--- |. $_ } split("\n",Dumper($json_no_data));
169177
my $msg = join("\n",
170178
qq|----------- REQUEST ----------|,
179+
" "x20 . qq|--- Method |.$self->req->method,
171180
" "x20 . qq|--- Route: |.$self->url_for,
172-
" "x20 . qq|--- Name: |.$self->current_route,
173-
" "x20 . qq|--- Params|,
181+
" "x20 . qq|--- Name: |.$self->current_route,
182+
" "x20 . qq|--- Params |,
174183
@plines,
175184
" "x20 . qq|--- JSON|,
176185
@jlines,

‎lib/Flair/Controller/FlairJob.pm‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@ sub default_sort {
1313
sub create ($self) {
1414

1515
# create a minion flair job and return the job id
16+
$self->log->trace("create flair job ",
17+
{ filter => \&Dumper, value =>$self->req->json});
1618

1719
# validate input, return error if not valid
1820
# $self->openapi->valid_input or return;
1921
if ( ! $self->openapi->valid_input ) {
20-
$self->log->error("Input from API is invalid");
2122
return;
2223
}
2324
my $input = $self->req->json;
24-
$self->log->trace("create flair job ", { filter => \&Dumper, value =>$input});
2525

2626
$self->log->debug("creating processor...");
2727
#$self->minion->add_task("flair_it" => sub ($job, @args) {
@@ -30,7 +30,11 @@ sub create ($self) {
3030
# my $proc = Flair::Processor->new(log => $self->log, db => $self->db);
3131
# $proc->do_flairing($job, @args);
3232
#});
33-
my $id = $self->minion->enqueue('flair_it', [$input]);
33+
my $flair_job_test = $self->config->{flair_job_test};
34+
$self->log->warn("flair job test is $flair_job_test");
35+
my $id = $self->minion->enqueue(
36+
'flair_it', [$input, $flair_job_test], {expire=>3600}
37+
);
3438
$self->log->debug("Minion Job id = $id enqueued");
3539

3640
# set status and allow openapi to validate output
@@ -40,6 +44,23 @@ sub create ($self) {
4044
);
4145
}
4246

47+
sub bulk_flair ($self) {
48+
if ( ! $self->openapi->valid_input ) {
49+
$self->log->error("Input from API is invalid");
50+
return;
51+
}
52+
my $input = $self->req->json;
53+
# { targets: [ {target_type: x, target_id: y }, ... ] }
54+
55+
my $id = $self->minion->enqueue('flair_bulk', [$input], {expire=> 3600});
56+
$self->log->debug("Bulk flair Mininon Job id = $id enqueued");
57+
58+
$self->render(
59+
status => 202,
60+
json => {job_id => $id},
61+
);
62+
}
63+
4364
sub list ($self) {
4465
$self->log->debug("listing jobs");
4566
# TODO: define filters ids=>[], notes=>[], queues=>[], states=>[], etc.

‎lib/Flair/Controller/Regex.pm‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ use strict;
55
use warnings;
66
use Data::Dumper::Concise;
77
use HTML::Entities;
8+
# add this when we can add modules to perl image
9+
# use Lingua::StopWords qw(getStopWords);
810
use Try::Tiny;
911
use Mojo::Base 'Flair::Controller::Api', -signatures;
1012
use Carp qw(longmess);
@@ -25,6 +27,13 @@ sub create ($self) {
2527
my $id = $self->db->regex->regex_exists($match);
2628

2729
if ( ! $id ) {
30+
if ($self->is_stop_word($match)) {
31+
$self->log->debug("Refusing to add a stopword $match to User defined entities");
32+
$self->render(
33+
status => 409,
34+
json => { error => "Match is a stop word" },
35+
);
36+
}
2837
my $href = $self->db->regex->create($json);
2938
$self->log->debug("href = ",{filter=>\&Dumper, value => $href});
3039

@@ -48,6 +57,13 @@ sub create ($self) {
4857
return;
4958
}
5059

60+
sub is_stop_word ($self, $match) {
61+
return undef;
62+
# when Lingua::StopWords is added to perl build we can do this
63+
# my $stopwords = getStopWords('en');
64+
# return grep { $stopwords->{$_} } $match;
65+
}
66+
5167
sub list ($self) {
5268

5369
# validate input, return error if not valid

‎lib/Flair/Images.pm‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package Flair::Images;
22

33
use Mojo::Base -base, -signatures;
4+
use Mojo::UserAgent;
45
use Data::Dumper::Concise;
56
use HTML::Element;
67
use MIME::Base64;
@@ -93,16 +94,21 @@ sub download_image ($self, $image) {
9394
my $ua = Mojo::UserAgent->new();
9495
$ua->proxy->detect;
9596
my $insec = $self->insec;
96-
my $asset = $ua->insecure($insec)->max_redirects(5)->get($link)->result->content->asset;
97-
my $md5 = md5_hex($asset->slurp);
98-
my $ext = $self->get_extension($link);
99-
my $newfile = $self->download_dir . "/$md5.$ext";
100-
if ( $self->file_exists($newfile) ) {
97+
my $result = $ua->insecure($insec)->max_redirects(5)->get($link)->result;
98+
99+
if ($result->is_success) {
100+
my $asset = $result->content->asset;
101+
my $md5 = md5_hex($asset->slurp);
102+
my $ext = $self->get_extension($link);
103+
my $newfile = $self->download_dir . "/$md5.$ext";
104+
if ( $self->file_exists($newfile) ) {
105+
return $newfile;
106+
}
107+
$asset->move_to($newfile);
108+
$self->log->debug("Downloaded $link to file $newfile");
101109
return $newfile;
102110
}
103-
$asset->move_to($newfile);
104-
$self->log->debug("Downloaded $link to file $newfile");
105-
return $newfile;
111+
return undef;
106112
}
107113

108114
sub get_extension ($self, $link) {
@@ -140,7 +146,7 @@ sub rewrite_img_element($self, $image, $new_uri) {
140146
}
141147

142148
sub get_alt ($self, $link, $element) {
143-
my $orig = $element->attr('alt');
149+
my $orig = $element->attr('alt') // '';
144150
my $new = ( $link =~ /^data:/ ) ? 'Cached copy of embedded data uri'
145151
: "Locally cached copy of $link";
146152
# $new .= "-$orig-" if defined $orig;

‎lib/Flair/Model/Apikeys.pm‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ sub update_sqlite ($self, $id, $update_href) {
7171
my $sql = $self->getSAL;
7272
my $where = { apikey_id => $id };
7373
my $href = dclone($update_href);
74-
$href->{'`key`'} = delete $href->{key};
74+
$href->{'`apikey`'} = delete $href->{apikey};
7575
my ($stmt, @bind) = $sql->update($self->tablename, $href, $where);
7676
$self->log_sql(__PACKAGE__, $stmt, @bind);
7777

@@ -87,7 +87,7 @@ sub patch_sqlite ($self, $id, $update_href) {
8787
my $sql = $self->getSAL;
8888
my $where = { apikey_id => $id };
8989
my $href = dclone($update_href);
90-
$href->{'`key`'} = delete $href->{key};
90+
$href->{'`apikey`'} = delete $href->{apikey};
9191
my ($stmt, @bind) = $sql->update($self->tablename,
9292
$href,
9393
$where);

0 commit comments

Comments
 (0)