auto-encode urls + scan any page/file for hrefs

include/util/config.hpp

@@ -21,7 +21,6 @@ namespace inst::config {
 	extern bool usbAck;
 	extern bool gayMode;
 	extern bool useSound;
-	extern bool encodeurl;
 	extern bool fixticket;
 	extern bool httpkeyboard;
 

romfs/lang/de.json

@@ -177,7 +177,6 @@
       "auto_update": "Suche automatisch nach Updates für TinWoo",
       "gay_option": "Design verwenden",
       "useSound": "Nutzen von Benachrichtungstönen während Installation",
-      "encodeurl": "Auto encode urls for http server installs",
       "fixticket": "Convert temporary tickets to permanent",
       "usehttpkeyboard": "Use keyboard during http server installs",
       "sig_url": "Signatur Patches URL: ",

romfs/lang/en.json

@@ -177,7 +177,6 @@
       "auto_update": "Check for updates to TinWoo Installer automatically",
       "gay_option": "Use Theme",
       "useSound": "Use sound notifications during installs",
-      "encodeurl": "Auto encode urls for http server installs",
       "fixticket": "Convert temporary tickets to permanent",
       "usehttpkeyboard": "Use keyboard during http server installs",
       "sig_url": "Signature patches source URL: ",

romfs/lang/es.json

@@ -177,7 +177,6 @@
       "auto_update": "Busca actualizaciones para el instalador de TinWoo automáticamente",
       "gay_option": "Usar tema",
       "useSound": "Usar notificaciones de sonido durante las instalaciones",
-      "encodeurl": "Usar archivos php antiguos para instalaciones de servidor http",
       "fixticket": "Convert temporary tickets to permanent",
       "usehttpkeyboard": "Use keyboard during http server installs",
       "sig_url": "URL de origen de parches de firma: ",

romfs/lang/fr.json

@@ -177,7 +177,6 @@
       "auto_update": "Vérifier les mises à jour de l'installateur TinWoo automatiquement",
       "gay_option": "Utiliser le thème",
       "useSound": "Utiliser les notifications sonores lors des installations",
-      "encodeurl": "Encodage automatique des URL pour les installations de serveur http",
       "fixticket": "Convertir des billets temporaires en billets permanents",
       "usehttpkeyboard": "Utiliser le clavier lors des installations du serveur http",
       "sig_url": "Patchs de signatures URL: ",

romfs/lang/it.json

@@ -177,7 +177,6 @@
       "auto_update": "Controlla aggiornamenti di TinWoo Installer automaticamente",
       "gay_option": "Usa tema",
       "useSound": "Use sound notifications during installs",
-      "encodeurl": "Auto encode urls for http server installs",
       "fixticket": "Convert temporary tickets to permanent",
       "usehttpkeyboard": "Use keyboard during http server installs",
       "sig_url": "Fonte URL SigPatches: ",

romfs/lang/jp.json

@@ -177,7 +177,6 @@
       "auto_update": "TinWooのアップデートを自動的に確認する",
       "gay_option": "テーマを使用する",
       "useSound": "インストール時にサウンドで通知する",
-      "encodeurl": "Auto encode urls for http server installs",
       "fixticket": "Convert temporary tickets to permanent",
       "usehttpkeyboard": "Use keyboard during http server installs",
       "sig_url": "署名パッチのソースURL: ",

romfs/lang/ru.json

@@ -177,7 +177,6 @@
       "auto_update": "Автоматически проверять наличие обновлений",
       "gay_option": "Использовать тему",
       "useSound": "Use sound notifications during installs",
-      "encodeurl": "Auto encode urls for http server installs",
       "fixticket": "Convert temporary tickets to permanent",
       "usehttpkeyboard": "Use keyboard during http server installs",
       "sig_url": "URL для скачивания Signature patches: ",

romfs/lang/tw.json

@@ -177,7 +177,6 @@
       "auto_update": "自動檢查Tinwoo Installer的更新版本",
       "gay_option": "套用主題",
       "useSound": "在安裝過程中使用音效提醒進度",
-      "encodeurl": "Auto encode urls for http server installs",
       "fixticket": "Convert temporary tickets to permanent",
       "usehttpkeyboard": "Use keyboard during http server installs",
       "sig_url": "簽名修補程式來源URL: ",

source/netInstall.cpp

@@ -47,7 +47,7 @@ SOFTWARE.
 #include "ui/instPage.hpp"
 
 const unsigned int MAX_URL_SIZE = 1024;
-const unsigned int MAX_URLS = 2048;
+const unsigned int MAX_URLS = 1000;
 const int REMOTE_INSTALL_PORT = 2000;
 static int m_serverSocket = 0;
 static int m_clientSocket = 0;
@@ -113,6 +113,15 @@ namespace netInstStuff {
 		return new_str;
 	}
 
+	std::string url_decode(const std::string& encoded)
+	{
+		int output_length;
+		const auto decoded_value = curl_easy_unescape(nullptr, encoded.c_str(), static_cast<int>(encoded.length()), &output_length);
+		std::string result(decoded_value, output_length);
+		curl_free(decoded_value);
+		return result;
+	}
+
 	void InitializeServerSocket()
 	{
 		try {
@@ -287,13 +296,9 @@ namespace netInstStuff {
 
 	std::vector<std::string> OnSelected()
 	{
-		/*
-		https://switchbrew.github.io/libnx/hid_8h.html#aa163470a1a7b811662e5c38905cc86fba4d9ae7fa7e27704abaf86c8a8a5398bd
-		*/
 		padConfigureInput(8, HidNpadStyleSet_NpadStandard);
 		PadState pad;
 		padInitializeAny(&pad);
-
 		u64 freq = armGetSystemTickFreq();
 		u64 startTime = armGetSystemTick();
 
@@ -301,11 +306,9 @@ namespace netInstStuff {
 
 		try {
 			ASSERT_OK(curl_global_init(CURL_GLOBAL_ALL), "Curl failed to initialized");
-
 			// Initialize the server socket if it hasn't already been
 			if (m_serverSocket == 0) {
 				InitializeServerSocket();
-
 				if (m_serverSocket <= 0)
 				{
 					THROW_FORMAT("Server socket failed to initialize.\n");
@@ -321,15 +324,15 @@ namespace netInstStuff {
 			LOG_DEBUG("%s %s\n", "Switch IP is ", ourIPAddress.c_str());
 			LOG_DEBUG("%s\n", "Waiting for network");
 			LOG_DEBUG("%s\n", "B to cancel");
-
 			std::vector<std::string> urls;
+			std::vector<std::string> tmp_array;
 
 			while (true) {
 				padUpdate(&pad);
 
 				// If we don't update the UI occasionally the Switch basically crashes on this screen if you press the home button
 				u64 newTime = armGetSystemTick();
-				
+
 				if (newTime - startTime >= freq * 0.25) {
 					startTime = newTime;
 					inst::ui::mainApp->CallForRender();
@@ -372,148 +375,125 @@ namespace netInstStuff {
 						inst::config::setConfig();
 						//refresh options page
 						inst::ui::mainApp->optionspage->setMenuText();
-
+						break;
 					}
 
 					else {
 						std::string response;
-						if (inst::util::formatUrlString(url) == "" || url == "https://" || url == "http://" || url == "HTTP://" || url == "HTTPS://")
+						if (inst::util::formatUrlString(url) == "" || url == "https://" || url == "http://" || url == "HTTP://" || url == "HTTPS://") {
 							inst::ui::mainApp->CreateShowDialog("inst.net.url.invalid"_lang, "", { "common.ok"_lang }, false, "romfs:/images/icons/fail.png");
+							break;
+						}
 						else {
-							if (url[url.size() - 1] != '/')
+							//if (url[url.size() - 1] != '/') //does this line even do anything?
 
-								//First try and stream the links
+							//First try and stream the links
-								response = inst::curl::downloadToBuffer(url);
+							response = inst::curl::downloadToBuffer(url);
 
 							//If the above fails we probably have an html page - try to download it instead.
 							if (response.empty()) {
 								response = inst::curl::html_to_buffer(url);
 							}
-
-							// debug - write the webpage to check for the game urls
-							/*
-							  FILE * fp;
-							  fp = fopen ("index.html", "a+");
-							  auto *info = response.c_str();
-							  fprintf(fp, "%s", info);
-							  fclose(fp);
-							*/
-							//end of debug
 						}
 
 						if (!response.empty()) {
-							if (response[0] == '{')
+							if (response[0] == '{') {
 								try {
 								nlohmann::json j = nlohmann::json::parse(response);
 								for (const auto& file : j["files"]) {
-									/* info on dealing with c++ vector
-									https://www.cplusplus.com/reference/vector/vector/
-									https://www.cplusplus.com/reference/vector/vector/push_back/
-									*/
 									urls.push_back(file["url"]);
 								}
 
-								/*
-								//debug scan http links for size of file
-								FILE * fp;
-								fp = fopen ("http link log.txt", "a+");
-
-								for (const auto &file : j["files"]) {
-									std::string url = file["url"];
-									std::string size = file["size"];
-									auto *info = url.c_str();
-									auto *info2 = size.c_str();
-									fprintf(fp, "%s-%sMB\n", info, info2);
-								}
-								fclose(fp);
-								//end of debug
-								//*/
-
 								return urls;
-							}
+								response.clear();
-							catch (const nlohmann::detail::exception& ex) {
+								}
+								catch (const nlohmann::detail::exception& ex) {
 								LOG_DEBUG("Failed to parse JSON\n");
+								}
 							}
-
+							else if (!response.empty()) {
-							else if (response[0] == '<') {
 								std::size_t index = 0;
 								while (index < response.size()) {
 									std::string link;
 									auto found = findCaseInsensitive(response, "href=\"", index);
-									if (found == std::string::npos)
+									if (found == std::string::npos) {
 										break;
+									}
 
 									index = found + 6;
 									while (index < response.size()) {
 										if (response[index] == '"') {
 											if (link.find("../") == std::string::npos)
 												if (findCaseInsensitive(link, ".nsp") != std::string::npos || findCaseInsensitive(link, ".nsz") != std::string::npos || findCaseInsensitive(link, ".xci") != std::string::npos || findCaseInsensitive(link, ".xcz") != std::string::npos) {
-													if (inst::config::encodeurl) {
+
-														// if url encoded is set in options - url encode the link			
+													/*
-														link = urlencode(link);
+													Try to see if the href links contain http - if not add the own url
-														// if the link doesn't contain http in the url add the url from the settings page
+													defined in the settings page
-														if (link.find("http") == std::string::npos) {
+													*/
-															std::string before_strip = stripfilename(url);
+													if (link.find("http") == std::string::npos) {
-															urls.push_back(before_strip + "/" + link);
+														std::string before_strip = stripfilename(url);
-														}
+														tmp_array.push_back(before_strip + "/" + link);
-														else {
-															urls.push_back(link);
-														}
 													}
 													else {
-														// if the link doesn't contain http in the url add the url from the settings page
+														tmp_array.push_back(link);
-														if (link.find("http") == std::string::npos) {
-															std::string before_strip = stripfilename(url);
-															urls.push_back(before_strip + "/" + link);
-														}
-														else {
-															urls.push_back(link);
-														}
 													}
 												}
 											break;
 										}
 										link += response[index++];
 									}
-
 								}
-								if (urls.size() > 0) {
+								if (tmp_array.size() > 0) {
+
+									//code to decode the url (if it's encoded), if not then (re)encode all urls.
+									for (unsigned long int i = 0; i < tmp_array.size(); i++) {
+										std::string debug = tmp_array[i];
+										std::string decoded = url_decode(debug);
+										debug = urlencode(decoded);
+										urls.push_back(debug);
+									}
 
-									//debug scan http links
 									/*
 									FILE * fp;
-										fp = fopen ("http link log.txt", "a+");
+									fp = fopen ("links.txt", "a+");
+									for (unsigned long int i = 0; i < urls.size(); i++) {
+										std::string x = urls[i];
+										const char *info = x.c_str();
+										fprintf(fp, "%s\n", info);
+									}
+									fclose(fp);
+									*/
 
-										for (unsigned long int i = 0; i < urls.size(); i++) {
+									tmp_array.clear(); //we may as well clear this now as it's done it's job..
-											std::string debug = urls[i];
-											const char *info = debug.c_str();
-											fprintf(fp, "%s\n", info);
-										}
-										fclose(fp);
-										*/
-										//end of debug
 									std::sort(urls.begin(), urls.end(), inst::util::ignoreCaseCompare);
 									return urls;
+									break;
 								}
 
 								else {
 									inst::ui::mainApp->CreateShowDialog("inst.net.url.nolinks"_lang, "", { "common.ok"_lang }, false, "romfs:/images/icons/fail.png");
+									LOG_DEBUG("Failed to parse games from HTML\n");
+									break;
 								}
-
+								response.clear();
-								LOG_DEBUG("Failed to parse games from HTML\n");
+							}
+							
+							else {
+								inst::ui::mainApp->CreateShowDialog("inst.net.index_error"_lang, "inst.net.index_error_info"_lang, { "common.ok"_lang }, true, "romfs:/images/icons/fail.png");
+								break;
 							}
 						}
 
 						else {
 							LOG_DEBUG("Failed to fetch game list\n");
 							inst::ui::mainApp->CreateShowDialog("inst.net.index_error"_lang, "inst.net.index_error_info"_lang, { "common.ok"_lang }, true, "romfs:/images/icons/fail.png");
+							break;
 						}
 					}
 				}
 
 				struct sockaddr_in client;
 				socklen_t clientLen = sizeof(client);
-
 				m_clientSocket = accept(m_serverSocket, (struct sockaddr*)&client, &clientLen);
 
 				if (m_clientSocket >= 0)
@@ -542,27 +522,24 @@ namespace netInstStuff {
 
 					while (std::getline(urlStream, segment, '\n')) urls.push_back(segment);
 					std::sort(urls.begin(), urls.end(), inst::util::ignoreCaseCompare);
-
 					break;
 				}
 				else if (errno != EAGAIN)
 				{
 					THROW_FORMAT("Failed to open client socket with code %u\n", errno);
+					break;
 				}
 			}
-
 			return urls;
-			}
 		}
-		
 		catch (std::runtime_error& e) {
-		close(m_serverSocket);
+			close(m_serverSocket);
-		m_serverSocket = 0;
+			m_serverSocket = 0;
-		LOG_DEBUG("Failed to perform remote install!\n");
+			LOG_DEBUG("Failed to perform remote install!\n");
-		LOG_DEBUG("%s", e.what());
+			LOG_DEBUG("%s", e.what());
-		fprintf(stdout, "%s", e.what());
+			fprintf(stdout, "%s", e.what());
-		inst::ui::mainApp->CreateShowDialog("inst.net.failed"_lang, (std::string)e.what(), { "common.ok"_lang }, true, "romfs:/images/icons/fail.png");
+			inst::ui::mainApp->CreateShowDialog("inst.net.failed"_lang, (std::string)e.what(), { "common.ok"_lang }, true, "romfs:/images/icons/fail.png");
-		return {};
+			return {};
 		}
 	}
 }

source/ui/optionsPage.cpp

@@ -152,10 +152,6 @@ namespace inst::ui {
 		useSoundOption->SetColor(COLOR("#FFFFFFFF"));
 		useSoundOption->SetIcon(this->getMenuOptionIcon(inst::config::useSound));
 		this->menu->AddItem(useSoundOption);
-		auto encodeurl = pu::ui::elm::MenuItem::New("options.menu_items.encodeurl"_lang);
-		encodeurl->SetColor(COLOR("#FFFFFFFF"));
-		encodeurl->SetIcon(this->getMenuOptionIcon(inst::config::encodeurl));
-		this->menu->AddItem(encodeurl);
 		auto fixticket = pu::ui::elm::MenuItem::New("options.menu_items.fixticket"_lang);
 		fixticket->SetColor(COLOR("#FFFFFFFF"));
 		fixticket->SetIcon(this->getMenuOptionIcon(inst::config::fixticket));
@@ -287,18 +283,6 @@ namespace inst::ui {
 						break;
 
 					case 7:
-						if (inst::config::encodeurl) {
-							inst::config::encodeurl = false;
-						}
-						else {
-							inst::config::encodeurl = true;
-						}
-						this->setMenuText();
-						this->menu->SetSelectedIndex(index);
-						inst::config::setConfig();
-						break;
-
-					case 8:
 						if (inst::config::fixticket) {
 							inst::config::fixticket = false;
 						}
@@ -310,7 +294,7 @@ namespace inst::ui {
 						inst::config::setConfig();
 						break;
 
-					case 9:
+					case 8:
 						if (inst::config::httpkeyboard) {
 							inst::config::httpkeyboard = false;
 						}
@@ -322,11 +306,11 @@ namespace inst::ui {
 						inst::config::setConfig();
 						break;
 
-					case 10:
+					case 9:
 						sigPatchesMenuItem_Click();
 						break;
 
-					case 11:
+					case 10:
 						keyboardResult = inst::util::softwareKeyboard("options.sig_hint"_lang, inst::config::sigPatchesUrl.c_str(), 500);
 						if (keyboardResult.size() > 0) {
 							inst::config::sigPatchesUrl = keyboardResult;
@@ -336,7 +320,7 @@ namespace inst::ui {
 						}
 						break;
 
-					case 12:
+					case 11:
 						keyboardResult = inst::util::softwareKeyboard("inst.net.url.hint"_lang, inst::config::httpIndexUrl.c_str(), 500);
 						if (keyboardResult.size() > 0) {
 							inst::config::httpIndexUrl = keyboardResult;
@@ -346,7 +330,7 @@ namespace inst::ui {
 						}
 						break;
 
-					case 13:
+					case 12:
 						languageList = languageStrings;
 						languageList.push_back("options.language.system_language"_lang);
 						rc = inst::ui::mainApp->CreateShowDialog("options.language.title"_lang, "options.language.desc"_lang, languageList, false, flag );
@@ -383,7 +367,7 @@ namespace inst::ui {
 						mainApp->FadeOut();
 						mainApp->Close();
 						break;
-					case 14:
+					case 13:
 						if (inst::util::getIPAddress() == "1.0.0.127") {
 							inst::ui::mainApp->CreateShowDialog("main.net.title"_lang, "main.net.desc"_lang, { "common.ok"_lang }, true, "romfs:/images/icons/update.png");
 							break;
@@ -395,7 +379,7 @@ namespace inst::ui {
 						}
 						this->askToUpdate(downloadUrl);
 						break;
-					case 15:
+					case 14:
 						inst::ui::mainApp->CreateShowDialog("options.credits.title"_lang, "options.credits.desc"_lang, { "common.close"_lang }, true, "romfs:/images/icons/credits.png");
 						break;
 					default:

source/util/config.cpp

@@ -18,7 +18,6 @@ namespace inst::config {
 	bool useSound;
 	bool usbAck;
 	bool validateNCAs;
-	bool encodeurl;
 	bool fixticket;
 	bool httpkeyboard;
 
@@ -37,7 +36,6 @@ namespace inst::config {
 			{"validateNCAs", validateNCAs},
 			{"httpIndexUrl", httpIndexUrl},
 			{"httplastUrl", httplastUrl},
-			{"encodeurl", encodeurl},
 			{"fixticket", fixticket},
 			{"httpkeyboard", httpkeyboard}
 		};
@@ -51,7 +49,6 @@ namespace inst::config {
 			nlohmann::json j;
 			file >> j;
 			autoUpdate = j["autoUpdate"].get<bool>();
-			encodeurl = j["encodeurl"].get<bool>();
 			fixticket = j["fixticket"].get<bool>();
 			httpkeyboard = j["httpkeyboard"].get<bool>();
 			deletePrompt = j["deletePrompt"].get<bool>();
@@ -78,7 +75,6 @@ namespace inst::config {
 			deletePrompt = true;
 			gayMode = false;
 			useSound = true;
-			encodeurl = false;
 			fixticket = true;
 			httpkeyboard = false;
 			ignoreReqVers = true;

source/util/curl.cpp

@@ -126,6 +126,8 @@ namespace inst::curl {
 	}
 
 	std::string html_to_buffer(const std::string ourUrl) {
+		//best to remove "temp.html if it already exists!"
+		if (std::filesystem::remove("temp.html"));
 		CURL* curl;
 		FILE* fp;
 		CURLcode res{};