PhantomJS | Scan site for Headers, Cookies, Domains, ContentSize

Pozdravljeni.

Spisal sem eno PhantomJS kodo za pregled strani.

Koda izpiše:
- Server header (lahko se doda dodatne na izpis)
- Vse URL-je, ki jih ob obisku strani brskalnik zlouda
- Group by po domenah (in prenešena velikost)
- Group by po ContentType (in prenešeni velikosti).

Program se požene z ukazom: phantomjs koda.js
Debug se omogoči: phantomjs --debug=true koda.js

PhantomJS Command Line Options: https://phantomjs.org/api/command-line

V spodnjem primeru se klic na domeno zemljevid.najdi.si ponovi 2x, čeprav v brskalniku ni tako. Debug izpiše:
Resource request error: QNetworkReply::NetworkError(OperationCanceledError) ( "Operation canceled" ) URL: " https://zemljevid.najdi.si/api/NSZemlje... "

Za izpis header-jev je obvezno potrebno dodati "/" na koncu domene.

Kilo/mega bajti se računajo na "pravilen" način; 1024=kilo, 1024*1024=mega. Tako je prav!


To je to, če komu pride prav. Se da pa še izboljšat...

Primer izpisa:



Koda:

// Phantom JS

// multiple sites check
var sites = [
	"https://www.gov.si/",
	"https://www.policija.si/",
	"https://www.ip-rs.si/",
	"https://www.kpk-rs.si/",
	"http://www.sodisce.si/"
];

// single site check
var sites = ["https://www.kclj.si/"];

// print only these headers
var printheaders = ["Server", "server"];

// regex
var regex1 = "^(?:https?:\/\/)?(?:[^@\/\n]+@)?(?:www\.)?([^:\/\n]+)";

// call site function
function callsite(index) {
	
	// print only first response
	var firstresponse = true;
	
	// list on resources
	var resources = [];
	
	// groupresources
	var groupresources = [];
	
	// contenttype
	var contenttype = [];
	
	// total content size
	var tcs = 0;
	
	// print current/total and site name
	console.log("[" + String(index + 1) + "/" + sites.length + "] " + sites[index]);
	
	// create dt now
	var now = Date.now();

	// create page
	var page = new WebPage();
	
	// set user agent
	page.settings.userAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/666";
	
	page.onLoadFinished = function() {

		var count = 0;
		var cookies = page.cookies;
		
		console.log(" [Cookies]");
		
		if (cookies.length == 0) {
			console.log("  Empty");
		};

		for(var cookie in cookies) {

			var dt = new Date(cookies[cookie].expires);
			var datediff = dt.getTime() - now;

			var expiresin = "";

			if (isNaN(datediff)) {
				expiresin = "Do konca seje";
			}
			else {
				if (datediff > 999) {
					var datesec = (datediff / 1000).toFixed(0);
					if (datesec > 59) {
						var datemin =(datesec / 60).toFixed(0);
						if (datemin > 59) {
							var datehour = (datemin / 60).toFixed(0);
							if (datehour > 23) {
								var datedays = (datehour / 24).toFixed(0);
								if (datedays > 30) {
									var datemonth = (datedays / 31).toFixed(0);
									expiresin = "months=" + datemonth;
								}
								else {
									expiresin = "days=" + datedays;
								};
							}
							else {
								expiresin = "hours=" + datehour;
							};
						}
						else {
							expiresin = "minutes=" + datemin;
						};
					}
					else {
						expiresin = "seconds=" + datesec;
					};
				}
				else {
					expiresin = "miliseconds=" + datediff;
				};
			};
			console.log("  Index = " + String(count) + " | Name = " + cookies[cookie].name + " | Expires = " + expiresin + " (" + cookies[cookie].expires + ")"  + " | Domain = " + cookies[cookie].domain);
			count = count + 1;
		};
		
		phantom.clearCookies();
		
		localStorage.clear();
		
		page.clearMemoryCache();
		page.close();
		
		// print all resources
		console.log(" [Resources]")
		for (i = 0; i < resources.length; i = i + 1) {
			
			// print resource
			console.log("  Index = " + String(i) + " | URL = " + String(resources[i][0]));
			
			// sum by domain
			var matches = resources[i][0].match(regex1);
			
			if (matches.length > 0) {
				
				isnew = true;
				for (j = 0; j < groupresources.length; j = j + 1) {
					if (groupresources[j][0] == matches[0]) {
						isnew = false;
						break;
					};
				};
				
				if (isnew == true) {
					//console.log("DEBUG " + resources[i][1]);
					groupresources.push([matches[0], 1, resources[i][1]]);
				}
				else {
					for (j = 0; j < groupresources.length; j = j + 1) {
						if (groupresources[j][0] == matches[0]) {
							count = groupresources[j][1] + 1;
							bsize = groupresources[j][2] + resources[i][1];
							groupresources.splice(j, 1);
							groupresources.push([matches[0], count, bsize]);
							break;
						};
					};
				};
				
			};
			
			
			// sum by content type
			ct = resources[i][2];
			cs = resources[i][1];

			if (ct.indexOf(";") > 0) {
				ct = ct.substring(0, ct.indexOf(";"));
			};
			
			isnew = true;
			
			for (j = 0; j < contenttype.length; j = j + 1) {
				if (contenttype[j][0] == ct) {
					isnew = false;
					break;
				};
			};
			
			if (isnew == true) {
				contenttype.push([ct, cs]);
			}
			else {
				for (j = 0; j < contenttype.length; j = j + 1) {
					if (contenttype[j][0] == ct) {
						cs = contenttype[j][1] + cs;
						contenttype.splice(j, 1);
						contenttype.push([ct, cs]);
						break;
					};
				};
			};
			
		};
		
		// print all group resources
		console.log(" [Group by Domain]")
		for (i = 0; i < groupresources.length; i = i + 1) {
			
			totalsizein = 0;
			if (groupresources[i][2] > (1024*1024)) {
				totalsizein = String(Math.round(groupresources[i][2] / (1024 * 1024))) + " MB";
			}
			else if (groupresources[i][2] > 1024) {
				totalsizein = String(Math.round(groupresources[i][2] / 1024)) + " KB";
			}
			else {
				totalsizein = String(groupresources[i][2]) + " B";
			};
			
			console.log("  Index = " + String(i) + " | Sum = " + String(groupresources[i][1]) + " | Size = " + String(totalsizein) + " (" + String(Math.round(groupresources[i][2] / tcs * 10000) / 100) + "%)" + " | Base URL = " + String(groupresources[i][0]));
		};
		
		// print all group resources
		console.log(" [Group by ContentType]")
		for (i = 0; i < contenttype.length; i = i + 1) {
			
			totalsizein = 0;
			if (contenttype[i][1] > (1024*1024)) {
				totalsizein = String(Math.round(contenttype[i][1] / (1024 * 1024))) + " MB";
			}
			else if (contenttype[i][1] > 1024) {
				totalsizein = String(Math.round(contenttype[i][1] / 1024)) + " KB";
			}
			else {
				totalsizein = String(contenttype[i][1]) + " B";
			};			
			
			console.log("  Index = " + String(i) + " | ContentSize = " + String(totalsizein) + " (" + String(Math.round(contenttype[i][1] / tcs * 10000) / 100) + "%)" + " | ContentType = " + String(contenttype[i][0]));
		};

		if (index < sites.length - 1) {
			index = index + 1;
			callsite(index);
		}
		else {
			console.log("Time to die.");
			phantom.exit(0);
		};
	};
	
	// print header(s)
	page.onResourceReceived = function (response) {

		// fakin duplicates
		if ((response.bodySize != null) && (response.contentType != null)) {
			resources.push([response.url, response.bodySize, response.contentType]);
			tcs = tcs + response.bodySize;
		};
		
		if (response.url == sites[index]) {
			
			if (firstresponse == true) {
				
				headers = response.headers;
				serverfound = false;
				console.log(" [Headers]");
				
				for (i = 0; i < headers.length; i = i + 1) {
					
					// find and print headers
					if (printheaders.includes(headers[i].name)) {
						console.log("  Name = " + String(headers[i].name) + " | Value = " + String(headers[i].value));
						serverfound = true;
					};
					
				};
				
				if (serverfound == false) {
					console.log("  Name = Server | Value = No value. Wat ar yu hiding!?");
				};
				
				firstresponse = false;
			};
		};
	};
	
	// open site
	page.open(sites[index], function() {
	});
	
};

// main entry
if (sites.length > 0) {
	callsite(0);
}
else {
	console.log("Empty list.");
	phantom.exit(0);
};

Čist sam tko, mimogrede,

// regex
var regex1

ali pa

// contenttype
var contenttype = [];

Teh commentov nihče na svetu ne rabi ker so totalno useless. Kaj mi pove comment če je dobesedno isti string kot ime spremenljivke. Raje prišparaj vrstico.

Ne kot da je to edina stvar ki tvojo kodo naredi težko berljivo, samo to je tak unopinionated objektiven komentar.