undo lightning block sync changes and remove HTTP-based sync support

Author: arik-so

lightning-block-sync/Cargo.toml

@@ -10,8 +10,6 @@ Utilities to fetch the chain data from a block source and feed them into Rust Li
 edition = "2018"
 
 [features]
-# Expose HTTP client necessary for incremental graph updates
-rapid-graph-sync = []
 rest-client = [ "serde", "serde_json", "chunked_transfer" ]
 rpc-client = [ "serde", "serde_json", "chunked_transfer" ]
 

lightning-block-sync/src/http.rs

@@ -96,24 +96,11 @@ impl<'a> std::net::ToSocketAddrs for &'a HttpEndpoint {
 	}
 }
 
-mod http_client {
-	use std::net::SocketAddr;
-
-	#[cfg(feature = "tokio")]
-	use tokio::net::TcpStream;
-	#[cfg(not(feature = "tokio"))]
-	use std::net::TcpStream;
-
-	/// Client for making HTTP requests.
-	pub struct HttpClient {
-		pub(super) address: SocketAddr,
-		pub(super) stream: TcpStream,
-	}
+/// Client for making HTTP requests.
+pub struct HttpClient {
+	address: SocketAddr,
+	stream: TcpStream,
 }
-#[cfg(feature = "rapid-graph-sync")]
-pub use http_client::HttpClient;
-#[cfg(not(feature = "rapid-graph-sync"))]
-pub(crate) use http_client::HttpClient;
 
 impl HttpClient {
 	/// Opens a connection to an HTTP endpoint.

lightning-graph-sync/Cargo.toml

@@ -13,9 +13,6 @@ Utility to fetch gossip routing data from LNSync or LNSync-like server
 
 [dependencies]
 lightning = { version = "0.0.104", path = "../lightning", features = ["rapid-graph-sync"] }
-lightning-block-sync = { version = "0.0.104", path = "../lightning-block-sync", features = ["rapid-graph-sync", "rest-client"] }
-tokio = { version = "1.0", features = [ "io-util", "net", "time" ], optional = true }
 
 [dev-dependencies]
 bitcoin = { version = "0.27", default-features = false, features = ["secp-recovery"] }
-tokio = { version = "1.0", features = [ "macros", "rt" ] }

lightning-graph-sync/src/lib.rs

@@ -11,33 +11,12 @@
 use std::fs::File;
 
 use lightning::routing::network_graph;
-use lightning_block_sync::http::*;
 
 use crate::error::GraphSyncError;
 
 pub mod error;
 pub mod processing;
 
-pub async fn sync_network_graph_with_url(
-	network_graph: &network_graph::NetworkGraph,
-	sync_host: &str,
-	sync_port: u16,
-	sync_path: &str,
-) -> Result<(), GraphSyncError> {
-	// make sure there is precisely one leading slash
-	let canonical_path = format!("/{}", sync_path.trim_start_matches("/"));
-	let http_endpoint = HttpEndpoint::for_host(sync_host.into()).with_port(sync_port);
-	let http_client_result = HttpClient::connect(&http_endpoint);
-	let mut http_client = http_client_result?;
-
-	let response_result = http_client
-		.get::<BinaryResponse>(&canonical_path, sync_host)
-		.await;
-
-	let response_bytes = response_result?.0;
-	processing::update_network_graph(&network_graph, &response_bytes[..])
-}
-
 pub fn sync_network_graph_with_file_path(
 	network_graph: &network_graph::NetworkGraph,
 	sync_path: &str,
@@ -54,264 +33,8 @@ mod tests {
 	use bitcoin::Network;
 
 	use lightning::routing::network_graph::NetworkGraph;
-	use lightning_block_sync::http::{BinaryResponse, HttpClient};
-
-	use crate::{sync_network_graph_with_file_path, sync_network_graph_with_url};
-	use crate::error::GraphSyncError;
-	use crate::tests::http_server_test_duplication::HttpServer;
-
-	mod http_server_test_duplication {
-		use std::io::{BufRead, Write};
-		use std::time::Duration;
-
-		use lightning_block_sync::http::HttpEndpoint;
-
-		/// Server for handling HTTP client requests with a stock response.
-		pub struct HttpServer {
-			address: std::net::SocketAddr,
-			handler: std::thread::JoinHandle<()>,
-			shutdown: std::sync::Arc<std::sync::atomic::AtomicBool>,
-		}
-
-		/// Body of HTTP response messages.
-		pub enum MessageBody<T: ToString> {
-			Empty,
-			Content(T),
-			ChunkedContent(T),
-		}
-
-		impl HttpServer {
-			fn responding_with_data(status: &str, data: &[u8]) -> Self {
-				let mut response_bytes = format!(
-					"{}\r\n\
-					 Content-Length: {}\r\n\
-					 \r\n\
-					 ",
-					status,
-					data.len()
-				)
-					.as_bytes()
-					.to_vec();
-				response_bytes.extend_from_slice(data);
-				HttpServer::responding_with(response_bytes)
-			}
-
-			pub fn responding_with_ok(data: &[u8]) -> Self {
-				HttpServer::responding_with_data("HTTP/1.1 200 OK", data)
-			}
-
-			pub fn responding_with_server_error(data: &[u8]) -> Self {
-				HttpServer::responding_with_data("HTTP/1.1 500 Internal Server Error", data)
-			}
-
-			fn responding_with(response_bytes: Vec<u8>) -> Self {
-				let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
-				let address = listener.local_addr().unwrap();
-
-				let shutdown = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
-				let shutdown_signaled = std::sync::Arc::clone(&shutdown);
-				let handler = std::thread::spawn(move || {
-					for stream in listener.incoming() {
-						let mut stream = stream.unwrap();
-						stream
-							.set_write_timeout(Some(Duration::from_secs(5)))
-							.unwrap();
-
-						let lines_read = std::io::BufReader::new(&stream)
-							.lines()
-							.take_while(|line| !line.as_ref().unwrap().is_empty())
-							.count();
-						if lines_read == 0 {
-							continue;
-						}
-
-						for chunk in response_bytes.chunks(16) {
-							if shutdown_signaled.load(std::sync::atomic::Ordering::SeqCst) {
-								return;
-							} else {
-								if let Err(_) = stream.write(chunk) {
-									break;
-								}
-								if let Err(_) = stream.flush() {
-									break;
-								}
-							}
-						}
-					}
-				});
-
-				Self {
-					address,
-					handler,
-					shutdown,
-				}
-			}
-
-			fn shutdown(self) {
-				self.shutdown
-					.store(true, std::sync::atomic::Ordering::SeqCst);
-				self.handler.join().unwrap();
-			}
-
-			pub fn endpoint(&self) -> HttpEndpoint {
-				HttpEndpoint::for_host(self.address.ip().to_string()).with_port(self.address.port())
-			}
-		}
-	}
-
-	#[tokio::test]
-	async fn test_mock_endpoint_fidelity() {
-		let valid_response = vec![
-			76, 68, 75, 2, 2, 174, 0, 0, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70,
-			174, 99, 247, 79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 8,
-			153, 192, 0, 2, 27, 0, 0, 2, 22, 7, 207, 206, 25, 164, 197, 231, 230, 231, 56, 102, 61,
-			250, 251, 187, 172, 38, 46, 79, 247, 108, 44, 155, 48, 219, 238, 252, 53, 192, 6, 67,
-			2, 36, 125, 157, 176, 223, 175, 234, 116, 94, 248, 201, 225, 97, 235, 50, 47, 115, 172,
-			63, 136, 88, 216, 115, 11, 111, 217, 114, 84, 116, 124, 231, 107, 2, 163, 216, 116,
-			204, 120, 152, 52, 172, 208, 143, 140, 71, 242, 209, 153, 111, 23, 61, 222, 28, 106,
-			198, 149, 87, 166, 201, 180, 51, 64, 252, 143, 166, 2, 57, 240, 15, 230, 190, 255, 224,
-			70, 176, 137, 155, 249, 125, 20, 170, 114, 97, 221, 142, 89, 231, 233, 205, 126, 199,
-			216, 149, 155, 0, 210, 45, 67, 174, 0, 0, 111, 226, 140, 10, 182, 241, 179, 114, 193,
-			166, 162, 70, 174, 99, 247, 79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0,
-			0, 0, 0, 10, 222, 162, 0, 8, 38, 0, 1, 2, 158, 1, 242, 121, 152, 106, 204, 131, 186,
-			35, 93, 70, 216, 10, 237, 224, 183, 89, 95, 65, 3, 83, 185, 58, 138, 181, 64, 187, 103,
-			127, 68, 50, 2, 201, 19, 17, 138, 136, 149, 185, 226, 156, 137, 175, 110, 32, 237, 0,
-			217, 90, 31, 100, 228, 149, 46, 219, 175, 168, 77, 4, 143, 38, 128, 76, 97, 2, 155, 9,
-			173, 229, 75, 82, 137, 57, 21, 112, 207, 99, 92, 145, 58, 48, 229, 10, 228, 163, 3,
-			106, 147, 39, 177, 90, 242, 186, 249, 11, 210, 54, 2, 212, 16, 225, 46, 42, 209, 254,
-			11, 28, 254, 196, 167, 139, 164, 241, 7, 176, 120, 192, 199, 148, 152, 38, 17, 118, 41,
-			122, 240, 223, 249, 158, 145, 2, 8, 153, 192, 0, 2, 27, 0, 0, 1, 27, 1, 0, 40, 0, 0, 0,
-			0, 0, 0, 3, 232, 0, 0, 0, 1, 0, 0, 0, 125, 0, 0, 0, 0, 58, 85, 116, 216, 10, 222, 162,
-			0, 8, 38, 0, 1, 1, 27, 1, 0, 40, 0, 0, 0, 0, 0, 0, 3, 232, 0, 0, 3, 232, 0, 0, 0, 1, 0,
-			0, 0, 0, 29, 129, 25, 192,
-		];
-		let server = HttpServer::responding_with_ok(&valid_response[..]);
-
-		let mut client = HttpClient::connect(&server.endpoint()).unwrap();
-		let binary_response = client
-			.get::<BinaryResponse>("/foo", "foo.com")
-			.await
-			.unwrap();
-		assert_eq!(binary_response.0, valid_response);
-	}
-
-	#[tokio::test]
-	async fn http_sync_fails_on_server_error() {
-		let valid_response = vec![
-			76, 68, 75, 2, 2, 174, 0, 0, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70,
-			174, 99, 247, 79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 8,
-			153, 192, 0, 2, 27, 0, 0, 2, 22, 7, 207, 206, 25, 164, 197, 231, 230, 231, 56, 102, 61,
-			250, 251, 187, 172, 38, 46, 79, 247, 108, 44, 155, 48, 219, 238, 252, 53, 192, 6, 67,
-			2, 36, 125, 157, 176, 223, 175, 234, 116, 94, 248, 201, 225, 97, 235, 50, 47, 115, 172,
-			63, 136, 88, 216, 115, 11, 111, 217, 114, 84, 116, 124, 231, 107, 2, 163, 216, 116,
-			204, 120, 152, 52, 172, 208, 143, 140, 71, 242, 209, 153, 111, 23, 61, 222, 28, 106,
-			198, 149, 87, 166, 201, 180, 51, 64, 252, 143, 166, 2, 57, 240, 15, 230, 190, 255, 224,
-			70, 176, 137, 155, 249, 125, 20, 170, 114, 97, 221, 142, 89, 231, 233, 205, 126, 199,
-			216, 149, 155, 0, 210, 45, 67, 174, 0, 0, 111, 226, 140, 10, 182, 241, 179, 114, 193,
-			166, 162, 70, 174, 99, 247, 79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0,
-			0, 0, 0, 10, 222, 162, 0, 8, 38, 0, 1, 2, 158, 1, 242, 121, 152, 106, 204, 131, 186,
-			35, 93, 70, 216, 10, 237, 224, 183, 89, 95, 65, 3, 83, 185, 58, 138, 181, 64, 187, 103,
-			127, 68, 50, 2, 201, 19, 17, 138, 136, 149, 185, 226, 156, 137, 175, 110, 32, 237, 0,
-			217, 90, 31, 100, 228, 149, 46, 219, 175, 168, 77, 4, 143, 38, 128, 76, 97, 2, 155, 9,
-			173, 229, 75, 82, 137, 57, 21, 112, 207, 99, 92, 145, 58, 48, 229, 10, 228, 163, 3,
-			106, 147, 39, 177, 90, 242, 186, 249, 11, 210, 54, 2, 212, 16, 225, 46, 42, 209, 254,
-			11, 28, 254, 196, 167, 139, 164, 241, 7, 176, 120, 192, 199, 148, 152, 38, 17, 118, 41,
-			122, 240, 223, 249, 158, 145, 2, 8, 153, 192, 0, 2, 27, 0, 0, 1, 27, 1, 0, 40, 0, 0, 0,
-			0, 0, 0, 3, 232, 0, 0, 0, 1, 0, 0, 0, 125, 0, 0, 0, 0, 58, 85, 116, 216, 10, 222, 162,
-			0, 8, 38, 0, 1, 1, 27, 1, 0, 40, 0, 0, 0, 0, 0, 0, 3, 232, 0, 0, 3, 232, 0, 0, 0, 1, 0,
-			0, 0, 0, 29, 129, 25, 192,
-		];
-		let server = HttpServer::responding_with_server_error(&valid_response[..]);
-
-		let block_hash = genesis_block(Network::Bitcoin).block_hash();
-		let network_graph = NetworkGraph::new(block_hash);
-
-		let before = network_graph.to_string();
-		assert_eq!(before.len(), 31);
-
-		let endpoint = &server.endpoint();
-		let sync_result = sync_network_graph_with_url(
-			&network_graph,
-			endpoint.host(),
-			endpoint.port(),
-			endpoint.path(),
-		)
-			.await;
-
-		let error_string = if let Err(GraphSyncError::IOError(error)) = sync_result {
-			error.to_string()
-		} else {
-			assert!(false);
-			"".to_string()
-		};
-
-		assert!(error_string.contains("status_code: 500,"));
 
-		let after = network_graph.to_string();
-		assert_eq!(after.len(), 31);
-	}
-
-	#[tokio::test]
-	async fn test_http_request() {
-		let valid_response = vec![
-			76, 68, 75, 2, 2, 0, 0, 8, 153, 192, 0, 2, 27, 0, 0, 2, 22, 7, 207, 206, 25, 164, 197,
-			231, 230, 231, 56, 102, 61, 250, 251, 187, 172, 38, 46, 79, 247, 108, 44, 155, 48, 219,
-			238, 252, 53, 192, 6, 67, 2, 36, 125, 157, 176, 223, 175, 234, 116, 94, 248, 201, 225,
-			97, 235, 50, 47, 115, 172, 63, 136, 88, 216, 115, 11, 111, 217, 114, 84, 116, 124, 231,
-			107, 0, 0, 10, 222, 162, 0, 8, 38, 0, 1, 2, 158, 1, 242, 121, 152, 106, 204, 131, 186,
-			35, 93, 70, 216, 10, 237, 224, 183, 89, 95, 65, 3, 83, 185, 58, 138, 181, 64, 187, 103,
-			127, 68, 50, 2, 201, 19, 17, 138, 136, 149, 185, 226, 156, 137, 175, 110, 32, 237, 0,
-			217, 90, 31, 100, 228, 149, 46, 219, 175, 168, 77, 4, 143, 38, 128, 76, 97, 2, 8, 153,
-			192, 0, 2, 27, 0, 0, 1, 27, 1, 0, 40, 0, 0, 0, 0, 0, 0, 3, 232, 0, 0, 0, 1, 0, 0, 0,
-			125, 0, 0, 0, 0, 58, 85, 116, 216, 10, 222, 162, 0, 8, 38, 0, 1, 1, 27, 1, 0, 40, 0, 0,
-			0, 0, 0, 0, 3, 232, 0, 0, 3, 232, 0, 0, 0, 1, 0, 0, 0, 0, 29, 129, 25, 192,
-		];
-		let server = HttpServer::responding_with_ok(&valid_response[..]);
-
-		let block_hash = genesis_block(Network::Bitcoin).block_hash();
-		let network_graph = NetworkGraph::new(block_hash);
-
-		let before = network_graph.to_string();
-		assert_eq!(before.len(), 31);
-
-		let endpoint = &server.endpoint();
-		let sync_result = sync_network_graph_with_url(
-			&network_graph,
-			endpoint.host(),
-			endpoint.port(),
-			endpoint.path(),
-		)
-			.await;
-
-		if !sync_result.is_ok() {
-			let error_string = match sync_result.as_ref().unwrap_err() {
-				GraphSyncError::IOError(error) => error.to_string(),
-				GraphSyncError::DecodeError(error) => error.to_string(),
-				GraphSyncError::LightningError(error) => (&error.err).to_string(),
-				GraphSyncError::ProcessingError(error) => error.to_string(),
-			};
-			println!("Error: {}", error_string);
-		}
-
-		assert!(sync_result.is_ok());
-
-		let after = network_graph.to_string();
-		assert_eq!(after.len(), 1737);
-		assert!(
-			after.contains("021607cfce19a4c5e7e6e738663dfafbbbac262e4ff76c2c9b30dbeefc35c00643:")
-		);
-		assert!(
-			after.contains("02247d9db0dfafea745ef8c9e161eb322f73ac3f8858d8730b6fd97254747ce76b:")
-		);
-		assert!(
-			after.contains("029e01f279986acc83ba235d46d80aede0b7595f410353b93a8ab540bb677f4432:")
-		);
-		assert!(
-			after.contains("02c913118a8895b9e29c89af6e20ed00d95a1f64e4952edbafa84d048f26804c61:")
-		);
-		assert!(after.contains("channels: [619737530008010752]"));
-		assert!(after.contains("channels: [783241506229452801]"));
-	}
+	use crate::{sync_network_graph_with_file_path};
 
 	#[test]
 	fn test_sync_from_file() {