[iec] loading media via HTTP is almost as fast as from SD now!

idolpx · idolpx · commit 3a41e4d9db70 · 2024-11-14T07:46:07.000-05:00
diff --git a/lib/device/iec/drive.cpp b/lib/device/iec/drive.cpp
@@ -1155,7 +1155,7 @@ bool iecDrive::sendFile()
 {
     uint32_t count = 0, startpos;
 
-    uint8_t buf[128];
+    uint8_t buf[256];
     uint16_t load_address = 0;
     uint16_t sys_address = 0;
 
diff --git a/lib/meatloaf/disk/d64.cpp b/lib/meatloaf/disk/d64.cpp
@@ -213,7 +213,7 @@ bool D64MStream::seekEntry(std::string filename)
 
             //Debug_printv("index[%d] track[%d] sector[%d] filename[%s] entry.filename[%.16s]", index, track, sector, filename.c_str(), entryFilename.c_str());
 
-            // Debug_printv("filename[%s] entry[%s]", filename.c_str(), entryFilename.c_str());
+            //Debug_printv("filename[%s] entry[%s]", filename.c_str(), entryFilename.c_str());
 
             if (filename == entryFilename) // Match exact
             {
@@ -257,8 +257,8 @@ bool D64MStream::seekEntry(uint16_t index)
     uint16_t sectorOffset = index / 8;
     uint16_t entryOffset = (index % 8) * 32;
 
-    // Debug_printv("----------");
-    // Debug_printv("index[%d] sectorOffset[%d] entryOffset[%d] entry_index[%d]", index, sectorOffset, entryOffset, entry_index);
+    //Debug_printv("----------");
+    //Debug_printv("index[%d] sectorOffset[%d] entryOffset[%d] entry_index[%d]", index, sectorOffset, entryOffset, entry_index);
 
     if (index == 0 || index != entry_index)
     {
@@ -297,7 +297,7 @@ bool D64MStream::seekEntry(uint16_t index)
             if (next_track == 0)
                 return false;
 
-            // Debug_printv("Follow link track[%d] sector[%d] entryOffset[%d]", next_track, next_sector, entryOffset);
+            //Debug_printv("Follow link track[%d] sector[%d] entryOffset[%d]", next_track, next_sector, entryOffset);
             if (!seekSector(next_track, next_sector, entryOffset))
                 return false;
         }
@@ -312,7 +312,8 @@ bool D64MStream::seekEntry(uint16_t index)
         next_sector = entry.next_sector;
     }
 
-    // Debug_printv("r[%d] file_type[%02X] file_name[%.16s]", r, entry.file_type, entry.filename);
+    //std::string e = mstr::toHex((uint8_t *)&entry, sizeof(entry));
+    //Debug_printv("file_type[%02X] file_name[%.16s] entry[%s]", entry.file_type, entry.filename, e.c_str());
 
     // if ( next_track == 0 && next_sector == 0xFF )
     entry_index = index + 1;
@@ -425,15 +426,15 @@ bool D64MStream::seekPath(std::string path)
     // return D64Image.seekFile(containerIStream, path);
     if (mstr::endsWith(path, "#")) // Direct Access Mode
     {
-        Debug_printv("Direct Access Mode track[1] sector[0] path[%s]", path.c_str());
+        //Debug_printv("Direct Access Mode track[1] sector[0] path[%s]", path.c_str());
         seekCalled = false;
         return seekSector(1, 0);
     }
     else if (seekEntry(path))
     {
         // auto entry = containerImage->entry;
         auto type = decodeType(entry.file_type).c_str();
-        Debug_printv("filename[%.16s] type[%s] start_track[%d] start_sector[%d]", entry.filename, type, entry.start_track, entry.start_sector);
+        //Debug_printv("filename[%.16s] type[%s] start_track[%d] start_sector[%d]", entry.filename, type, entry.start_track, entry.start_sector);
 
         // Calculate file size
         uint8_t t = entry.start_track;
@@ -443,7 +444,7 @@ bool D64MStream::seekPath(std::string path)
         // Set position to beginning of file
         bool r = seekSector(t, s);
 
-        Debug_printv("blocks[%d] size[%d] available[%d] r[%d]", entry.blocks, _size, available(), r);
+        //Debug_printv("blocks[%d] size[%d] available[%d] r[%d]", entry.blocks, _size, available(), r);
 
         return r;
     }
diff --git a/lib/meatloaf/meat_media.h b/lib/meatloaf/meat_media.h
@@ -219,11 +219,11 @@ class ImageBroker {
             // Are we at the root of the pathInStream?
             if ( newFile->pathInStream == "")
             {
-                Debug_printv("DIRECTORY [%s]", url.c_str());
+                //Debug_printv("DIRECTORY [%s]", url.c_str());
             }
             else
             {
-                Debug_printv("SINGLE FILE [%s]", url.c_str());
+                //Debug_printv("SINGLE FILE [%s]", url.c_str());
             }
 
             repo.insert(std::make_pair(url, newStream));
diff --git a/lib/meatloaf/network/http.cpp b/lib/meatloaf/network/http.cpp
@@ -220,17 +220,19 @@ bool MeatHttpClient::HEAD(std::string dstUrl) {
     return rc;
 }
 
-bool MeatHttpClient::processRedirectsAndOpen(int range) {
+bool MeatHttpClient::processRedirectsAndOpen(uint32_t position, uint32_t size) {
     wasRedirected = false;
-    _size = -1;
 
     //Debug_printv("reopening url[%s] from position:%d", url.c_str(), range);
-    lastRC = openAndFetchHeaders(lastMethod, range);
+    lastRC = openAndFetchHeaders(lastMethod, position, size);
+
+    if (lastRC == 206)
+        isFriendlySkipper = true;
 
     while(lastRC == HttpStatus_MovedPermanently || lastRC == HttpStatus_Found || lastRC == 303)
     {
         //Debug_printv("--- Page moved, doing redirect to [%s]", url.c_str());
-        lastRC = openAndFetchHeaders(lastMethod, range);
+        lastRC = openAndFetchHeaders(lastMethod, position, size);
         wasRedirected = true;
     }
     
@@ -242,9 +244,9 @@ bool MeatHttpClient::processRedirectsAndOpen(int range) {
 
     _is_open = true;
     _exists = true;
-    _position = 0;
+    _position = position;
 
-    Debug_printv("size[%d] avail[%d] isFriendlySkipper[%d] isText[%d] httpCode[%d] method[%d]", _size, available(), isFriendlySkipper, isText, lastRC, lastMethod);
+    //Debug_printv("size[%d] avail[%d] isFriendlySkipper[%d] isText[%d] httpCode[%d] method[%d]", _size, available(), isFriendlySkipper, isText, lastRC, lastMethod);
 
     return true;
 }
@@ -274,30 +276,32 @@ void MeatHttpClient::setOnHeader(const std::function<int(char*, char*)> &lambda)
 }
 
 bool MeatHttpClient::seek(uint32_t pos) {
-    if(pos==_position)
-        return true;
 
     if(isFriendlySkipper) {
 
         if (_is_open) {
             // Read to end of the stream
             //Debug_printv("Skipping to end!");
-            char c[256];
-            while( esp_http_client_read(_http, c, 256) > 0 );
+            while(1)
+            {
+                char c[HTTP_BLOCK_SIZE];
+                int bytes = esp_http_client_read(_http, c, HTTP_BLOCK_SIZE);
+                if ( bytes < HTTP_BLOCK_SIZE )
+                    break;
+            }
         }
 
         bool op = processRedirectsAndOpen(pos);
 
         //Debug_printv("SEEK in HttpIStream %s: range request RC=%d", url.c_str(), lastRC);
-        
+
         if(!op)
             return false;
 
          // 200 = range not supported! according to https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
         if(lastRC == 206)
         {
             //Debug_printv("Seek successful");
-
             _position = pos;
             return true;
         }
@@ -345,20 +349,24 @@ uint32_t MeatHttpClient::read(uint8_t* buf, uint32_t size) {
 
     if (!_is_open) {
         Debug_printv("Opening HTTP Stream!");
-        processRedirectsAndOpen(0);
+        processRedirectsAndOpen(0, size);
     }
 
     if (_is_open) {
         //Debug_printv("Reading HTTP Stream!");
         auto bytesRead= esp_http_client_read(_http, (char *)buf, size );
         
-        if(bytesRead>0) {
+        if (bytesRead >= 0) {
             _position+=bytesRead;
+
+            if (bytesRead < size)
+                openAndFetchHeaders(lastMethod, _position);
         }
-        if(bytesRead<0)
+        if (bytesRead < 0)
             return 0;
 
-        return bytesRead;        
+        //Debug_printv("size[%d] bytesRead[%d] _position[%d]", size, bytesRead, _position);
+        return bytesRead;
     }
     return 0;
 };
@@ -378,34 +386,32 @@ uint32_t MeatHttpClient::write(const uint8_t* buf, uint32_t size) {
     return 0;
 };
 
-int MeatHttpClient::openAndFetchHeaders(esp_http_client_method_t meth, int resume) {
+int MeatHttpClient::openAndFetchHeaders(esp_http_client_method_t method, uint32_t position, uint32_t size) {
 
     if ( url.size() < 5)
         return 0;
 
     // Set URL and Method
     mstr::replaceAll(url, " ", "%20");
     esp_http_client_set_url(_http, url.c_str());
-    esp_http_client_set_method(_http, meth);
+    esp_http_client_set_method(_http, method);
 
     // Set Headers
     for (const auto& pair : headers) {
         std::cout << pair.first << ": " << pair.second << std::endl;
         esp_http_client_set_header(_http, pair.first.c_str(), pair.second.c_str());
     }
 
+    // Set Range Header
+    char str[40];
+    snprintf(str, sizeof str, "bytes=%lu-%lu", position, (position + size + 5));
+    esp_http_client_set_header(_http, "Range", str);
+    //Debug_printv("seeking range[%s] url[%s]", str, url.c_str());
+
     // POST
     // const char *post_data = "{\"field1\":\"value1\"}";
     // esp_http_client_set_post_field(client, post_data, strlen(post_data));
 
-
-    if(resume > 0) {
-        char str[40];
-        snprintf(str, sizeof str, "bytes=%lu-", (unsigned long)resume);
-        esp_http_client_set_header(_http, "range", str);
-        //Debug_printv("seeking range[%s]", str);
-    }
-
     //Debug_printv("--- PRE OPEN");
     esp_err_t rc = esp_http_client_open(_http, 0); // or open? It's not entirely clear...
 
@@ -417,7 +423,7 @@ int MeatHttpClient::openAndFetchHeaders(esp_http_client_method_t meth, int resum
         if(_size == -1 && lengthResp > 0) {
             // only if we aren't chunked!
             _size = lengthResp;
-            _position = 0;
+            _position = position;
         }
     }
 
@@ -433,6 +439,7 @@ esp_err_t MeatHttpClient::_http_event_handler(esp_http_client_event_t *evt)
         case HTTP_EVENT_ERROR: // This event occurs when there are any errors during execution
             Debug_printv("HTTP_EVENT_ERROR");
             meatClient->_error = 1;
+            meatClient->close();
             break;
 
         case HTTP_EVENT_ON_CONNECTED: // Once the HTTP has been connected to the server, no data exchange has been performed
@@ -447,15 +454,25 @@ esp_err_t MeatHttpClient::_http_event_handler(esp_http_client_event_t *evt)
             // Does this server support resume?
             // Accept-Ranges: bytes
 
-            if (mstr::equals("Accept-Ranges", evt->header_key, false))
+            // if (mstr::equals("Accept-Ranges", evt->header_key, false))
+            // {
+            //     if(meatClient != nullptr) {
+            //         meatClient->isFriendlySkipper = mstr::equals("bytes", evt->header_value,false);
+            //         Debug_printv("Accept-Ranges: %s",evt->header_value);
+            //     }
+            // }
+            if (mstr::equals("Content-Range", evt->header_key, false))
             {
+                //Debug_printv("Content-Range: %s",evt->header_value);
                 if(meatClient != nullptr) {
-                    meatClient->isFriendlySkipper = mstr::equals("bytes", evt->header_value,false);
-                    //Debug_printv("* Ranges info present '%s', comparison=%d!",evt->header_value, strcmp("bytes", evt->header_value)==0);
+                    meatClient->isFriendlySkipper = true;
+                    auto cr = util_tokenize(evt->header_value, '/');
+                    if( cr.size() > 1 )
+                        meatClient->_size = std::stoi(cr[1]);
                 }
             }
             // what can we do UTF8<->PETSCII on this stream?
-            else if (mstr::equals("Content-Type", evt->header_key, false))
+            if (mstr::equals("Content-Type", evt->header_key, false))
             {
                 std::string asString = evt->header_value;
                 bool isText = mstr::isText(asString);
diff --git a/lib/meatloaf/network/http.h b/lib/meatloaf/network/http.h
@@ -28,7 +28,7 @@
 class MeatHttpClient {
     esp_http_client_handle_t _http = nullptr;
     static esp_err_t _http_event_handler(esp_http_client_event_t *evt);
-    int openAndFetchHeaders(esp_http_client_method_t meth, int resume = 0);
+    int openAndFetchHeaders(esp_http_client_method_t method, uint32_t position, uint32_t size = HTTP_BLOCK_SIZE);
     esp_http_client_method_t lastMethod;
     std::function<int(char*, char*)> onHeader = [] (char* key, char* value){ 
         //Debug_printv("HTTP_EVENT_ON_HEADER, key=%s, value=%s", key, value);
@@ -81,7 +81,7 @@ class MeatHttpClient {
     bool PUT(std::string url);
     bool HEAD(std::string url);
 
-    bool processRedirectsAndOpen(int range);
+    bool processRedirectsAndOpen(uint32_t position, uint32_t size = HTTP_BLOCK_SIZE);
     bool open(std::string url, esp_http_client_method_t meth);
     void close();
     void setOnHeader(const std::function<int(char*, char*)> &f);

Original file line number	Diff line number	Diff line change
`@@ -1155,7 +1155,7 @@ bool iecDrive::sendFile()`
`1155`	`1155`	`{`
`1156`	`1156`	`uint32_t count = 0, startpos;`
`1157`	`1157`
`1158`		`- uint8_t buf[128];`
	`1158`	`+ uint8_t buf[256];`
`1159`	`1159`	`uint16_t load_address = 0;`
`1160`	`1160`	`uint16_t sys_address = 0;`
`1161`	`1161`
Original file line number	Diff line number	Diff line change
`@@ -219,11 +219,11 @@ class ImageBroker {`
`219`	`219`	`// Are we at the root of the pathInStream?`
`220`	`220`	`if ( newFile->pathInStream == "")`
`221`	`221`	`{`
`222`		`- Debug_printv("DIRECTORY [%s]", url.c_str());`
	`222`	`+ //Debug_printv("DIRECTORY [%s]", url.c_str());`
`223`	`223`	`}`
`224`	`224`	`else`
`225`	`225`	`{`
`226`		`- Debug_printv("SINGLE FILE [%s]", url.c_str());`
	`226`	`+ //Debug_printv("SINGLE FILE [%s]", url.c_str());`
`227`	`227`	`}`
`228`	`228`
`229`	`229`	`repo.insert(std::make_pair(url, newStream));`