Differences

This shows you the differences between two versions of the page.

--- web_browsers:chrome_screen_scraper:chrome_screen_scraper [2016/07/11 09:56] – peter
+++ web_browsers:chrome_screen_scraper:chrome_screen_scraper [2020/04/15 11:46] (current) – removed peter
@@ Line 1: / Line 1: @@
-====== Web Browsers - Chrome Screen Scraper ======
-Composed of two parts.
-  * Parser
-  * SaverApp
-===== Load the two extension files into Chrome =====
-To load these into chrome, you will need to go to the settings menu, choose more tools and then extensions.
-  * Tick the “Developer Mode” box that you see, this will allow you to load unpacked extensions.
-  * Click on the button “Load unpacked extension” and navigate to the directory where the Parser files exist.
-  * Click on the button “Load unpacked extension” and navigate to the directory where the SaveApp files exist.
-{{:web_browsers:chrome_screen_scraper:chrome_screen_scraper_extensions.png?800|}}
-Once this has been completed, you will see a new icon appear as shown below
-{{:web_browsers:chrome_screen_scraper:chrome_new_icon.png?300|}}
-This will be used to start the page parsing.
-Firstly though, you will need to launch the App, this is done by clicking on the “Launch” link as shown below
-{{:web_browsers:chrome_screen_scraper:chrome_launch_data_saver.png?500|}}
-You need to configure the 2 parts so that they can communicate with each other, this is done by the IDs that are given to each of the APP/Extension.
-You should now see a new window appear
-{{:web_browsers:chrome_screen_scraper:chrome_message_listener.png?500|}}
-You can click in the “my Application ID” box and copy the ID from there (just swoop the mouse over it whilst holding the left button down so that it selects it in blue and then CTRL+C to copy it.
-You can now click the Speech mark button
-{{:web_browsers:chrome_screen_scraper:chrome_new_icon.png?300|}}
-To launch the Data Parser Extension
-{{:web_browsers:chrome_screen_scraper:chrome_data_parser.png?500|}}
-===== Parser =====
-==== index.html ====
-<file html index.html>
-<!DOCTYPE html>
-<html>
-<head>
-  <meta charset="utf-8">
-  <link href="main.css" rel="stylesheet">
-</head>
-<body>
-  <h3>Data Parser</h3>
-  <label>Parser ID is <input id="appid" type="text"  readonly></input></label>
-  <div>
-    <label for="sendId">Destination ID</label>
-    <input id="sendId" type="text"></input>
-    <button id="saveID">Save ID</button>
-    <button id="process">Start Process</button>
-  </div>
-  <div id="log"></div>
-  <script src="index.js"></script>
-</body>
-</html>
-</file>
-==== main.css ====
-<file css main.css>
-input[type="text"] {
-  width: 240px;
-}
-#log {
-  background-color: rgb(226, 226, 250);
-  padding: 10px 20px;
-  margin-top: 10px;
-  height: 300px;
-  border: 1px solid black;
-  overflow-y: scroll;
-  overflow-x: hidden;
-}
-</file>
-==== index.js ====
-<file javascript index.js>
-(function(context){
-  document.getElementById("appid").value=chrome.runtime.id;
-  var remoteAppID ="";
-  var action ="";
-  var pageIndex=null;
-  console.log('Starting');
-  var sendId;
-  sendId=document.getElementById("sendId");
-  chrome.storage.local.get('remoteappid',function (result) {
-    console.log(result);
-    remoteAppID=result.remoteappid;
-    console.log('In Loop' +remoteAppID);
-    sendId.value=remoteAppID;
-  });
-  var logField = document.getElementById("log");
-  var save=document.getElementById("saveID");
-  var process=document.getElementById("process");
-  save.addEventListener('click', function() {
-    var remoteID=sendId.value;
-    appendLog("Saved");
-    chrome.storage.local.set({'remoteappid': remoteID});
-  });
-  process.addEventListener('click', function() {
-    pageIndex=1;
-    chrome.tabs.query(
-      { currentWindow: true, active: true },
-      function (tabArray) {
-        var activeTabId= tabArray[0];
-        chrome.tabs.sendMessage(activeTabId.id, {text: 'report_back',pageIndex: pageIndex}, doStuffWithDom);
-      }
-    );
-  });
-  function doStuffWithDom(res) {
-    var domContent=res.domContent;
-    var pageIndex=res.pageIndex;
-    //console.log('I received the following DOM content:\n' + domContent);
-    chrome.runtime.sendMessage(
-      sendId.value,
-      {myCustomMessage: JSON.stringify(domContent),myAction : "save",myIndex : pageIndex},
-      function(response) {
-        appendLog("Remote Message : " + response.result);
-        if (response.Success=="false"){
-          console.log("Remote Error : " + response.result);
-          appendLog("Remote Error : " + response.result);
-        }else{
-          chrome.tabs.query(
-            { currentWindow: true, active: true },
-            function (tabArray) {
-              var activeTabId= tabArray[0];
-              chrome.tabs.sendMessage(activeTabId.id, {text: 'process_next'},dealWithNextButton);
-            }
-          );
-          console.log(response.result);
-          appendLog(response.result);
-        }
-      })
-    }
-    chrome.runtime.onMessageExternal.addListener(
-      function(request, sender, sendResponse) {
-        appendLog("Remote Message : " + request.myResultAction);
-        if (request.result=="false"){
-          console.log("Remote Error : " + request.myResultAction);
-          appendLog("Remote Error : " + request.myResultAction);
-        }else{
-          chrome.tabs.query(
-            { currentWindow: true, active: true },
-            function (tabArray) {
-              var activeTabId= tabArray[0];
-              chrome.tabs.sendMessage(activeTabId.id, {text: 'process_next'},dealWithNextButton);
-            }
-          );
-          console.log(request.result);
-          appendLog(request.result);
-        }
-      }
-    );
-    function dealWithNextButton(res){
-      var result = res.success;
-      if (result==false){
-        appendLog("Finished.");
-        pageIndex=null;
-      }else{
-        appendLog("Still Processing.");
-      }
-    }
-    chrome.webNavigation.onDOMContentLoaded.addListener(function (details) {
-      if (pageIndex !=null){
-        pageIndex+=1;
-        chrome.tabs.query(
-          { currentWindow: true, active: true },
-          function (tabArray) {
-            var activeTabId= tabArray[0];
-            chrome.tabs.sendMessage(activeTabId.id, {text: 'report_back',pageIndex: pageIndex}, doStuffWithDom);
-          }
-        );
-      }
-    }
-  );
-  function doInCurrentTab(tabCallback) {
-  }
-  var appendLog = function(message) {
-    logField.innerText+="\n"+message;
-  };
-  context.appendLog = appendLog;
-})(window)
-</file>
-==== content.js ====
-<file javascript content.js>
-  // Listen for messages
-  chrome.runtime.onMessage.addListener(function (msg, sender, sendResponse) {
-    console.log('got here');
-    // If the received message has the expected format...
-    if (msg.text === 'report_back') {
-      // Call the specified callback, passing
-      // the web-page's DOM content as argument
-      var tbl = document.getElementsByName('table1')[0].innerHTML;
-      //sendResponse(document.all[0].outerHTML);
-      var rows = [];
-      var result = [];
-      $('table[name=table1]>tbody>tr').each(function(id){
-        var row = {'id': id+1};
-        if ($(this).find('td').length!=0){
-          $(this).find('td').each(function(index){
-            row[index] = $(this).text();
-          });
-          result.push(row);
-        }
-      });
-      var retResult = ({'domContent': result,'tableContent': result,'pageIndex':msg.pageIndex});
-      sendResponse(retResult);
-    }
-    if (msg.text === 'process_next') {
-      var nextButton = $('[name="nextButton"]');
-      if (nextButton.length>0){
-        nextButton.trigger( "click" );
-        var retResultS = ({'success':true});
-        sendResponse(retResultS);
-      }else{
-        var retResultF = ({'success':false});
-        sendResponse(retResultF);
-      }
-    }
-  });
-</file>
-==== eventPage.js ====
-<file javascript eventPage.js>
-var blacklistedIds = ["none"];
-chrome.runtime.onMessageExternal.addListener(
-  function(request, sender, sendResponse) {
-    appendLog("MSG RCV : " + request.myResultAction+' ' +request.myResultIndex);
-    if (request.myResultAction=='Ok Saved :') {
-      appendLog(request.myResultAction+' ' +request.myResultIndex);
-      sendResponse({"result":"Ok, got your message"});
-    } else {
-      sendResponse({"result":"Ops, I don't understand this message :" + request.myResultAction});
-    }
-  }
-);
-</file>
-==== mainfest.json ====
-<file json manifest.json>
-{
-  "name": "AG Data Parser Extension",
-  "version": "1.1",
-  "description": "Extension to parse pages and send them to the save sink.",
-  "browser_action": {
-    "default_title": "Send message to other apps",
-    "default_icon": "icon_16.png",
-    "default_popup": "index.html"
-  },
-  "background": {
-    "scripts": ["eventPage.js"],
-    "persistent": false
-  },
-  "content_scripts": [{
-    "matches": ["<all_urls>"],
-    "js": ["content.js","jquery.js"],
-    "run_at": "document_end"
-  }],
-  "permissions": [ "activeTab", "notifications","storage","webNavigation"],
-  "manifest_version": 2
-}
-</file>
-===== Save Sink Events =====
-index.html
-<file html index.html>
-<!DOCTYPE html>
-<html>
-<head>
-  <meta charset="utf-8">
-  <link href="main.css" rel="stylesheet">
-</head>
-<body>
-  <h3>Data Parser</h3>
-  <label>Parser ID is <input id="appid" type="text"  readonly></input></label>
-  <div>
-    <label for="sendId">Destination ID</label>
-    <input id="sendId" type="text"></input>
-    <button id="saveID">Save ID</button>
-    <button id="process">Start Process</button>
-  </div>
-  <div id="log"></div>
-  <script src="index.js"></script>
-</body>
-</html>
-</file>
-===== main.css =====
-<file css main.css>
-input[type="text"] {
-  width: 240px;
-}
-#log {
-  background-color: rgb(226, 226, 250);
-  padding: 10px 20px;
-  margin-top: 10px;
-  height: 300px;
-  border: 1px solid black;
-  overflow-y: scroll;
-  overflow-x: hidden;
-}
-</file>
-==== index.js ====
-<file javascript index.js>
-(function(context){
-  document.getElementById("appid").value=chrome.runtime.id;
-  var remoteAppID ="";
-  chrome.storage.local.get('remoteappid',function (result) {
-    remoteAppID=result.remoteappid;
-    sendId.value=remoteAppID;
-  });
-  var logField = document.getElementById("log");
-  var selectSave=document.getElementById("selectSave");
-  var sendId=document.getElementById("sendId");
-  var clearLog=document.getElementById("clearLog");
-  var saveId=document.getElementById("saveId");
-  saveId.addEventListener('click', function() {
-    var remoteID=sendId.value;
-    appendLog("Saved");
-    chrome.storage.local.set({'remoteappid': remoteID});
-    //alert (remoteID);
-  });
-  var chosenFileEntry=null;
-  clearLog.addEventListener('click', function() {
-    logField.innerText='';
-  });
-  errorHandler = function (obj) {
-    sendResponse({"result":"Something Went Wrong."});
-    console.log(obj);
-  };
-  selectSave.addEventListener('click', function() {
-    chrome.fileSystem.chooseEntry({type: 'saveFile',
-      suggestedName: 'output.csv'},
-      function(writableFileEntry) {
-        chosenFileEntry=writableFileEntry;
-        writableFileEntry.createWriter(function(writer) {
-          writer.seek(0);
-          writer.truncate(0);
-          writer.onwriteend = function(e) {
-          };
-          writer.write(new Blob(['0123456789'],
-            {type: 'text/plain'}
-          ));
-        }, errorHandler);
-      });
-    });
-    chrome.runtime.onMessageExternal.addListener(
-      function(request, sender, sendResponse) {
-        if (request.myAction) {
-          appendLog("Action from "+sender.id+": "+request.myAction);
-          //appendLog("Message from "+sender.id+": "+request.myCustomMessage);
-          if (request.myAction=='save' && chosenFileEntry==null){
-            sendResponse({"result":"No Save File Configured","Success":"false"});
-          }
-          if (request.myAction=='save' && chosenFileEntry!=null){
-            var arrData = typeof JSONData != 'object' ? JSON.parse(request.myCustomMessage) : request.myCustomMessage;
-            var CSV = '';
-            for (var i = 0; i < arrData.length; i++) {
-              var row = "";
-              for (var index in arrData[i]) {
-                //row += '"' + arrData[i][index] + '",';
-                row +=  arrData[i][index] + ',';
-              }
-              row.slice(0, row.length - 1);
-              CSV += row + '\r\n';
-            }
-            appendLog("Saving CSV : ");
-            chosenFileEntry.file(function(file) {
-              var reader = new FileReader();
-              reader.onload = function(e) {
-                var contents =e.target.result;
-                if (request.myIndex==1){
-                  contents='';
-                  appendLog("Clearing Contents");
-                }
-                appendLog("Saving Index : " + request.myIndex);//+CSV);
-                CSV=contents+CSV;
-                chrome.fileSystem.getWritableEntry(chosenFileEntry,
-                  function(writableFileEntry) {
-                    writableFileEntry.createWriter(function(writer) {
-                      writer.onwriteend = function(e) {
-                        appendLog("Save Complete - Sending Message");
-                        sendReply("true","Ok Saved :",request.myIndex);
-                      };
-                      chosenFileEntry.file(function(file) {
-                        writer.write(new Blob([CSV],
-                          {type: 'text/plain'}));
-                      });
-                    }, errorHandler);
-                  });
-                };
-                reader.readAsText(file);
-              });
-            }
-          } else {
-            sendResponse({"result":"Ops, I don't understand this message :" + request});
-          }
-        });
-        function sendReply(myResult,myResultAction,pageIndex){
-          chrome.runtime.sendMessage(
-            sendId.value,
-            {result: myResult, myResultAction : myResultAction, myResultIndex : pageIndex},
-            function(response) {
-              console.log("response: "+JSON.stringify(response));
-              appendLog("response: "+JSON.stringify(response));
-            }
-          )
-        }
-  var appendLog = function(message) {
-    logField.innerText+="\n"+message;
-  };
-  context.appendLog = appendLog;
-})(window)
-</file>
-==== manifest.json ====
-<file json manifest.json>
-{
-  "manifest_version": 2,
-  "name": "AG Data Saver Application",
-  "description": "Application to manage the save sink events from the parser.",
-  "version": "1.1",
-  "minimum_chrome_version": "23",
-  "icons": {
-    "16": "icon_16.png"
-  },
-  "app": {
-    "background": {
-      "scripts": ["main.js"]
-    }
-  },
-  "permissions": [{"fileSystem": ["write", "retainEntries", "directory"]},"storage"]
-}
-</file>