Browse Source

further refactoring

master
asiekierka 6 years ago
parent
commit
2f157b865c
7 changed files with 41 additions and 33 deletions
  1. 1
    1
      engines/danbooru.js
  2. 1
    1
      engines/derpibooru.js
  3. 1
    1
      engines/deviantart.js
  4. 1
    1
      engines/gelbooru.js
  5. 1
    1
      engines/moebooru.js
  6. 1
    1
      engines/pixiv-nekomaid.js
  7. 35
    27
      main.js

+ 1
- 1
engines/danbooru.js View File

@@ -13,7 +13,7 @@ module.exports = function(options) {

return this.URL + url;
},
parsePage: function(json, callback) {
parsePage: function(url, json, callback) {
var engine = this;
var images = _.map(json, function(image) {
var data = {

+ 1
- 1
engines/derpibooru.js View File

@@ -15,7 +15,7 @@ module.exports = function(options) {

return this.URL + url;
},
parsePage: function(json, callback) {
parsePage: function(url, json, callback) {
var images = _.map(json, function(image) {
var data = {
id: image.id_number,

+ 1
- 1
engines/deviantart.js View File

@@ -18,7 +18,7 @@ module.exports = function(options) {

return url;
},
parsePage: function(window, $, callback) {
parsePage: function(url, window, $, callback) {
var engine = this;
var urls = [], images = [];
$("span a.thumb").each(function() {

+ 1
- 1
engines/gelbooru.js View File

@@ -15,7 +15,7 @@ module.exports = function(options) {

return this.URL + url;
},
parsePage: function(window, $, callback) {
parsePage: function(url, window, $, callback) {
var images = [];
var engine = this;
$("img.preview").each(function() {

+ 1
- 1
engines/moebooru.js View File

@@ -12,7 +12,7 @@ module.exports = function(options) {

return this.URL + url;
},
parsePage: function(json, callback) {
parsePage: function(url, json, callback) {
var engine = this;
var images = _.map(json, function(image) {
return {

+ 1
- 1
engines/pixiv-nekomaid.js View File

@@ -13,7 +13,7 @@ module.exports = function(options) {

return url;
},
parsePage: function(json, callback) {
parsePage: function(url, json, callback) {
var images = _.chain(json.artworks)
.filter(function(image) {
return image.photos.length == 1; // Block multiple-image pixiv entries (FOR NOW)

+ 35
- 27
main.js View File

@@ -82,7 +82,6 @@ function usage() {
function downloadPage(position, callback) {
var listUrl = engine.getPageURL({page: position, tags: tags});
console.log("Downloading page #" + position);
if(params["d"] == "true") console.log("URL: " + listUrl);
if(_.has(engine, "downloadPage")) engine.downloadPage(listUrl, callback)
else request({url: listUrl, headers: {
"User-Agent": USER_AGENT
@@ -90,7 +89,7 @@ function downloadPage(position, callback) {
if(err) throw err;
if(response.statusCode != 200) {
console.log("Page " + listUrl + " returned error " + response.statusCode + "!");
} else callback(body);
} else callback(listUrl, body);
}
);
}
@@ -160,6 +159,20 @@ if(!_.isArray(tags) || tags.length < 1) {
}
}

function parsePageEngine(url, out, engine, callback) {
var parseFormat = engine.parseFormat || "html";
if(parseFormat == "html") {
jsdom.env({html: out, done: function(err, window) {
if(err) throw err;
engine.parsePage(url, window, jquery(window), callback);
}});
} else if(parseFormat == "json") {
engine.parsePage(url, JSON.parse(out), callback);
} else if(parseFormat == "raw") {
engine.parsePage(url, out, callback);
}
}

function updateWithParams(links, params) {
links = _.filter(links, function(data) {
if(_.isNumber(data.id) && _.isNumber(params["m"]) && data.id < params["m"])
@@ -197,39 +210,34 @@ function updateWithParams(links, params) {
return links;
}

var pid = 0;

function downloadImageArray(links, result, outDir, callback) {
if(links.length == 0) {
console.log("Downloaded all images, quitting");
return;
}
links = updateWithParams(links, params);
console.log(" " + links.length + " images found");
if(_.isObject(result)) {
// Update based on engine output
if(_.isNumber(result.pageChangeAmount)) pid += result.pageChangeAmount;
}
downloadImages(links, outDir, callback);
}

function downloadTagBased() {
console.log("Downloading with tags: " + tags.join(", "));
var outDir = params["O"] || tags.join(" ");
if(!fs.existsSync(outDir)) fs.mkdirSync(outDir);
var pid = 0;
var parseFunc = function(err, links, result) {
if(err) throw err;
else if(links.length == 0) {
console.log("Downloaded all images, quitting");
return;
}
links = updateWithParams(links, params);
console.log(" " + links.length + " images found");
if(_.isObject(result)) {
// Update based on engine output
if(_.isNumber(result.pageChangeAmount)) pid += result.pageChangeAmount;
}
downloadImages(links, outDir, downloadFunc);
if(err) throw err;
else downloadImageArray(links, result, outDir, downloadFunc);
};

var downloadFunc = function() {
downloadPage(pid, function(out) {
var parseFormat = engine.parseFormat || "html";
if(parseFormat == "html") {
jsdom.env({html: out, done: function(err, window) {
if(err) throw err;
engine.parsePage(window, jquery(window), parseFunc);
}});
} else if(parseFormat == "json") {
engine.parsePage(JSON.parse(out), parseFunc);
} else if(parseFormat == "raw") {
engine.parsePage(out, parseFunc);
}
downloadPage(pid, function(url, out) {
parsePageEngine(url, out, engine, parseFunc);
});
};
downloadFunc();

Loading…
Cancel
Save