@@ -18,7 +18,7 @@ public class WebManager {
1818 public void download (String url , OnDownloadListener downloadListener ) throws IOException {
1919 HttpsURLConnection connection = null ;
2020 BufferedReader reader = null ;
21-
21+
2222 try {
2323 URL urlObj = new URL (url );
2424 connection = (HttpsURLConnection ) urlObj .openConnection ();
@@ -35,24 +35,24 @@ public void download(String url, OnDownloadListener downloadListener) throws IOE
3535
3636 int contentLength = connection .getContentLength ();
3737 downloadListener .onDownloadStart (contentLength );
38-
38+
3939 StringBuilder content = new StringBuilder ();
4040 int totalRead = 0 ;
41-
41+
4242 try (InputStream is = connection .getInputStream ();
43- InputStreamReader isr = new InputStreamReader (is , charset );
44- BufferedReader br = new BufferedReader (isr )) {
45-
43+ InputStreamReader isr = new InputStreamReader (is , charset );
44+ BufferedReader br = new BufferedReader (isr )) {
45+
4646 char [] buffer = new char [8192 ];
4747 int bytesRead ;
48-
48+
4949 while ((bytesRead = br .read (buffer )) != -1 ) {
5050 content .append (buffer , 0 , bytesRead );
5151 totalRead += bytesRead ;
5252 downloadListener .onDownload (totalRead , contentLength );
5353 }
5454 }
55-
55+
5656 downloadListener .onDownloadFinished (content .toString ());
5757 } catch (IOException e ) {
5858 if (downloadListener != null ) {
@@ -61,7 +61,10 @@ public void download(String url, OnDownloadListener downloadListener) throws IOE
6161 throw e ;
6262 } finally {
6363 if (reader != null ) {
64- try { reader .close (); } catch (IOException ignored ) {}
64+ try {
65+ reader .close ();
66+ } catch (IOException ignored ) {
67+ }
6568 }
6669 if (connection != null ) {
6770 connection .disconnect ();
@@ -71,13 +74,26 @@ public void download(String url, OnDownloadListener downloadListener) throws IOE
7174
7275 public List <String > extractJsScripts (String html ) {
7376 List <String > scripts = new ArrayList <>();
74- Matcher matcher = Pattern .compile ("<script\\ b[^>]*>(.*?)</script>" ,
75- Pattern .CASE_INSENSITIVE | Pattern .DOTALL ).matcher (html );
77+ Matcher matcher =
78+ Pattern .compile ("<script>(.*?)</script>" , Pattern .CASE_INSENSITIVE | Pattern .DOTALL )
79+ .matcher (html );
7680
7781 while (matcher .find ()) {
78- String scriptContent = matcher .group (1 )
79- .replaceAll ("<!\\ [CDATA\\ [|\\ ]\\ ]>" , "" )
80- .trim ();
82+ String scriptContent = matcher .group (1 ).trim ();
83+ if (!scriptContent .isEmpty ()) {
84+ scripts .add (scriptContent );
85+ }
86+ }
87+ return scripts ;
88+ }
89+ public List <String > extractJsScriptLinks (String html ) {
90+ List <String > scripts = new ArrayList <>();
91+ Matcher matcher =
92+ Pattern .compile ("<script src=\" (.*)\" >[\\ s]+</script>" , Pattern .CASE_INSENSITIVE | Pattern .DOTALL )
93+ .matcher (html );
94+
95+ while (matcher .find ()) {
96+ String scriptContent = matcher .group (1 ).trim ();
8197 if (!scriptContent .isEmpty ()) {
8298 scripts .add (scriptContent );
8399 }
@@ -87,10 +103,13 @@ public List<String> extractJsScripts(String html) {
87103
88104 public interface OnDownloadListener {
89105 void onDownloadStart (int pageLength );
106+
90107 void onDownload (int downloaded , int pageLength );
108+
91109 void onDownloadFinished (String downloaded );
110+
92111 default void onDownloadError (IOException e ) {
93112 e .printStackTrace ();
94113 }
95114 }
96- }
115+ }
0 commit comments