Skip to content

Commit 32148a1

Browse files
happymanhappyman
committed
#65 work queue 使用 beanstalkd 並上傳 worker code
1 parent 14db2ef commit 32148a1

33 files changed

+214
-639
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,6 @@ dist
55
twmap_gen/config-hybridauth.php
66
twmap_gen/vendor
77
twmap3/config.inc.php
8+
twmap_gen/worker/make_map/vendor
89
docs/prominence/src
910
package-lock.json

‎twmap3/css/twmap3_main.css

+2-2
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ html {
3232
#opContainer {
3333
position:relative;
3434
margin-top:10px;
35-
margin-left:-200px
35+
margin-left:-200px;
3636
right: -500px;
3737
}
3838
#less {
@@ -155,7 +155,7 @@ textarea:hover {
155155
#marker_reload {
156156
cursor: pointer;
157157
}
158-
// google dropdown menu of mapstyle select
158+
/* google dropdown menu of mapstyle select */
159159
.gm-style-mtc > div:first-child {
160160
font-size:11px;
161161
padding: 0px;

‎twmap3/js/main.js

+2-3
Original file line numberDiff line numberDiff line change
@@ -453,9 +453,8 @@ var Geo2016_Options = {
453453
};
454454

455455
var DebugTile_Options = {
456-
getTileUrl: function (a, b) {
457-
return "//tile.happyman.idv.tw/map/debug/" + a.x + "/" + a.y + "/" + b + ".png";
458-
//eturn "https://chart.apis.google.com/chart?chst=d_text_outline&chs=256x256&chf=bg,s,00000044&chld=FFFFFF|32|h|000000|b|||x="+a.x+"|y="+a.y+"|z="+b+"||||__";
456+
getTileUrl: function (a, z) {
457+
return "//tile.happyman.idv.tw/map/debug/" + z + "/" + a.x + "/" + a.y + ".png";
459458
},
460459
tileSize: new google.maps.Size(256, 256),
461460
//maxZoom: 19,

‎twmap_gen/VERSION

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
3.02
1+
3.03

‎twmap_gen/api/made.php

+5-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,11 @@ function finish_task($param) {
4646
my_error_out("寫入資料庫失敗,請回報 $outimage");
4747
}
4848
// 最後搬移到正確目錄
49-
map_migrate($out_root, $uid, $mid);
49+
sleep(1);
50+
$ret = map_migrate($out_root, $uid, $mid);
51+
if ($ret == false) {
52+
msglog('error migrate directory');
53+
}
5054
// 寫入資料庫
5155
make_map_log($mid, $log_channel, $_REQUEST['agent'], $_REQUEST['params']);
5256

‎twmap_gen/backend_make.php

+19-9
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
}
1212

1313
require_once ("config.inc.php");
14-
require_once('lib/memqueue.php');
1514
ini_set("memory_limit", "512M");
1615
ini_set("max_execution_time", "3600");
1716
ignore_user_abort(true);
@@ -221,25 +220,36 @@
221220
);
222221
msglog($cmd_param);
223222

223+
// check api/made.php
224224
// uid limit r_flag xx yy shiftx shifty datum version outx outy title outimage ip channel
225225
$add_param_array=[ $MYUID, $user['limit'], $recreate_flag, $xx, $yy, $shiftx, $shifty, isset($inp['97datum'])? "97":"67",$version, $outx, $outy, $title,$outimage, $_SERVER['REMOTE_ADDR'],$log_channel ];
226226
// ouch forget to consider ipv6 addr
227227
$add_param_str=json_encode($add_param_array);
228228

229229
memcached_set($log_channel, $add_param_str);
230230

231+
232+
use xobotyi\beansclient\Client as beansclient;
233+
use xobotyi\beansclient\Socket\SocketsSocket as beansconnect;
234+
231235
if (isset($CONFIG['use_queue']) && $CONFIG['use_queue'] == true){
232-
$workload = $cmd_param;
233-
$mq = new Memqueue("localhost",11211);
234-
$id = $mq->push("mqq_make_map",$workload);
235-
236-
while(!$mq->is_processed('mqq_make_map',$id)) {
236+
// 使用 beanstalkd 請 config.inc.php 中加上相關參數
237+
$sock = new beansconnect(host: $CONFIG['beanstalk_server'], port: $CONFIG['beanstalk_port'], connectionTimeout: 2);
238+
$client = new beansclient(socket: $sock, defaultTube: $CONFIG['beanstalk_tube'], defaultTTR: 3600);
239+
$job = $client->put($cmd_param);
240+
while(1){
241+
$st = $client->statsJob($job['id']);
242+
if (is_null($st) || $st['state'] == 'reserved') {
243+
notify_web($log_channel,array('worker is working...'));
244+
$sock->disconnect();
245+
exit(0);
246+
}
247+
notify_web($log_channel,array("waiting for a worker "));
237248
sleep(2);
238-
notify_web($log_channel,array("waiting for queue worker "));
239249
}
240-
notify_web($log_channel,array('worker is working...'));
241-
250+
242251
} else {
252+
// 前端可能無法等太久
243253
exec("php cmd_make2.php ".$cmd_param, $output, $ret);
244254
if ($ret != 0) {
245255
foreach ($output as $line) {

‎twmap_gen/cloudprint.php

-81
This file was deleted.

‎twmap_gen/cmd_make2.php

+12-23
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@
4444
if (isset($opt['S'])) $jumpstop = $jump+1; else $jumpstop = 0;
4545
$remote_ip = isset($opt['i'])? $opt['i'] : "localhost";
4646
if (isset($opt['d'])) $debug_flag= 1; else $debug_flag = 0;
47-
if ($version != 1 && $version != 3 && $version != 2016)
48-
$version = 3;
47+
// default 2016, remove version 1
48+
if (!in_array($version, array(3,2016))) $version=2016;
4949
if (isset($opt['l'])) $log_channel = $opt['l']; else $log_channel = "";
5050
if (isset($opt['logurl_prefix']))
5151
$logurl_prefix=$opt['logurl_prefix'];
@@ -91,25 +91,8 @@
9191
else
9292
$type = determine_type($shiftx, $shifty);
9393

94-
// add version 3
95-
switch($version){
96-
case 1:
97-
/*
98-
if ($ph == 1 ) {
99-
cli_error_out("無澎湖圖資");
100-
}
101-
$g = new STB($stbpath, $startx, $starty, $shiftx, $shifty, $datum, $tmpdir);
102-
break;
103-
*/
104-
case 3:
105-
$g = new STB2($stbpath, $startx, $starty, $shiftx, $shifty, $ph, $datum, $tmpdir);
106-
$g->version = 3;
107-
break;
108-
case 2016:
109-
$g = new STB2($stbpath, $startx, $starty, $shiftx, $shifty, $ph, $datum, $tmpdir);
110-
$g->version = 2016;
111-
break;
112-
}
94+
$g = new STB2("", $startx, $starty, $shiftx, $shifty, $ph, $datum, $tmpdir);
95+
$g->version=$version;
11396

11497
if (isset($opt['G'])) {
11598
$g->include_gpx = 1;
@@ -345,8 +328,14 @@
345328
// log
346329
if (isset($opt['agent']))
347330
$url.="&agent=".trim($opt['agent']);
348-
cli_msglog("call $url");
349-
$output = file_get_contents($url);
331+
cli_msglog("call callback api");
332+
// $output = file_get_contents($url);
333+
try {
334+
$output = request_curl($url);
335+
} catch (Exception $e) {
336+
// 連不上 or 有意外
337+
cli_error_out('callback failed '.$url);
338+
}
350339
error_log(print_r($output,true));
351340
}
352341
cli_msglog("ps%100");

‎twmap_gen/composer.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
"license": "MIT",
66
"require" : {
77
"hybridauth/hybridauth" : "~3.0",
8-
"smarty/smarty": "4.3.1"
8+
"smarty/smarty": "4.3.1",
9+
"xobotyi/beansclient": "dev-master"
910
}
1011
}

‎twmap_gen/config.inc.php.sample

+6-7
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,6 @@ $gdal_dsn = sprintf("PG:dbname='%s' user='%s' password='%s' host='%s' port='%s'"
6969
require_once(__ROOT__."lib/twmapdb.inc.php");
7070
/* login */
7171
if (php_sapi_name() != "cli") {
72-
require_once(__ROOT__."lib/xuite.php");
73-
$fb_appid = '';
7472
/* template */
7573
$smarty = new Smarty();
7674
// composer autoload require_once(__ROOT__."lib/smarty/Smarty.class.php");
@@ -83,9 +81,7 @@ if (php_sapi_name() != "cli") {
8381

8482
$smarty->assign("twmap_gen_version", $twmap_gen_version);
8583
$smarty->assign("browser_url", $TWMAP3URL );
86-
$smarty->assign("fb_appid", $fb_appid );
8784
$smarty->assign('twmap3_html_root',$TWMAP3_html_root);
88-
8985
}
9086

9187
#
@@ -94,13 +90,16 @@ if (php_sapi_name() != "cli") {
9490
define('GEARMAN_SERVER','127.0.0.1');
9591
$CONFIG['use_gearman'] = false;
9692

97-
# check backend_make and lib/memqueue.php
93+
# check backend_make
94+
# 產生地圖 queue
9895
$CONFIG['use_queue'] = false;
96+
$CONFIG['beanstalk_server'] = 'localhost';
97+
$CONFIG['beanstalk_port'] = '11300';
98+
$CONFIG['beanstalk_tube'] = 'make_map';
9999

100-
define('APE_HOST','0.ape.tile.happyman.idv.tw');
101100
$CONFIG['admin'] = array( 3 );
102101
// gdalwarp -t_srs EPSG:4326 dem_20m.tif dem_20m-latlong.tif
103102
// source: http://data.gov.tw/node/35430
104-
define('twDEM_path', __ROOT__ ."db/DEM/dem_20m-latlong.tif");
103+
// define('twDEM_path', __ROOT__ ."db/DEM/dem_20m-latlong.tif");
105104
$CONFIG['poi_file'] = "/home/happyman/poi/MOI_OSM_Taiwan_TOPO_Rudy.poi";
106105

‎twmap_gen/css/twmap.css

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{ font-family: Verdana; font-size: 1em; }
1+
body { font-family: Verdana; font-size: 1em; min-width: 1000px;}
22
label { width: 10em; float: left; }
33
label.error { float: none; color: red; padding-left: .5em; vertical-align: top; }
44
p { clear: both; }
@@ -8,6 +8,7 @@ img { image-rendering: -moz-crisp-edges; -ms-interpolation-mode:bicubic; }
88
.fbshare {
99
cursor:pointer;
1010
}
11+
1112
#mapbrowse { width: 100%; }
1213
#browse { height: 600px; }
1314
#greeting {
@@ -75,7 +76,7 @@ background: #f5f3e5 url(http://code.jquery.com/ui/1.8.23/themes/south-street/ima
7576
border-radius: 10px;
7677
border: 2px solid #8AC007;
7778
padding: 10px;
78-
height: 130px;
79+
height: 110px;
7980
}
8081
/* drawer.css */
8182
.drawer {

0 commit comments

Comments
 (0)