上传文件至 'lib'

lib/drT.js

@@ -0,0 +1,189 @@
+// drT.js
+// 2022/09/30 write by hjdhnx
+// Licensed under the MIT license.
+
+(function () {
+    "use strict";
+
+    var drT = {
+        name: "drT",
+        version: "1.0.0",
+        templateSettings: {
+            evaluate:    /\{\{([\s\S]+?(\}?)+)\}\}/g,
+            interpolate: /\{\{([\s\S]+?)\}\}/g, // 变量渲染
+            encode:      /\{\{@([\s\S]+?)\}\}/g, // 变量自动url编码
+            use:         /\{\{#([\s\S]+?)\}\}/g,
+            useParams:   /(^|[^\w$])def(?:\.|\[[\'\"])([\w$\.]+)(?:[\'\"]\])?\s*\:\s*([\w$\.]+|\"[^\"]+\"|\'[^\']+\'|\{[^\}]+\})/g,
+            define:      /\{\{##\s*([\w\.$]+)\s*(\:|=)([\s\S]+?)#\}\}/g,
+            defineParams:/^\s*([\w$]+):([\s\S]+)/,
+            conditional: /\{\{\?(\?)?\s*([\s\S]*?)\s*\}\}/g, // ? if ?? else if ?? else
+            iterate:     /\{\{~\s*(?:\}\}|([\s\S]+?)\s*\:\s*([\w$]+)\s*(?:\:\s*([\w$]+))?\s*\}\})/g,
+            varname:	"fl",
+            strip:		true,
+            append:		true,
+            selfcontained: false,
+            doNotSkipEncoded: false
+        },
+        template: undefined, //fn, compile template
+        compile:  undefined, //fn, for express
+        log: true
+    }, _globals;
+
+    drT.encodeHTMLSource = function(doNotSkipEncoded) {
+        var encodeHTMLRules = { "&": "&#38;", "<": "&#60;", ">": "&#62;", '"': "&#34;", "'": "&#39;", "/": "&#47;" },
+            matchHTML = doNotSkipEncoded ? /[&<>"'\/]/g : /&(?!#?\w+;)|<|>|"|'|\//g;
+        return function(code) {
+            return code ? code.toString().replace(matchHTML, function(m) {return encodeHTMLRules[m] || m;}) : "";
+        };
+    };
+
+    _globals = (function(){ return this || (0,eval)("this"); }());
+
+    /* istanbul ignore else */
+    if (typeof module !== "undefined" && module.exports) {
+        module.exports = drT;
+    } else if (typeof define === "function" && define.amd) {
+        define(function(){return drT;});
+    } else {
+        _globals.drT = drT;
+    }
+
+    var startend = {
+        append: { start: "'+(",      end: ")+'",      startencode: "'+encodeHTML(" },
+        split:  { start: "';out+=(", end: ");out+='", startencode: "';out+=encodeHTML(" }
+    }, skip = /$^/;
+
+    function resolveDefs(c, block, def) {
+        return ((typeof block === "string") ? block : block.toString())
+            .replace(c.define || skip, function(m, code, assign, value) {
+                if (code.indexOf("def.") === 0) {
+                    code = code.substring(4);
+                }
+                if (!(code in def)) {
+                    if (assign === ":") {
+                        if (c.defineParams) value.replace(c.defineParams, function(m, param, v) {
+                            def[code] = {arg: param, text: v};
+                        });
+                        if (!(code in def)) def[code]= value;
+                    } else {
+                        new Function("def", "def['"+code+"']=" + value)(def);
+                    }
+                }
+                return "";
+            })
+            .replace(c.use || skip, function(m, code) {
+                if (c.useParams) code = code.replace(c.useParams, function(m, s, d, param) {
+                    if (def[d] && def[d].arg && param) {
+                        var rw = (d+":"+param).replace(/'|\\/g, "_");
+                        def.__exp = def.__exp || {};
+                        def.__exp[rw] = def[d].text.replace(new RegExp("(^|[^\\w$])" + def[d].arg + "([^\\w$])", "g"), "$1" + param + "$2");
+                        return s + "def.__exp['"+rw+"']";
+                    }
+                });
+                var v = new Function("def", "return " + code)(def);
+                return v ? resolveDefs(c, v, def) : v;
+            });
+    }
+
+    function unescape(code) {
+        return code.replace(/\\('|\\)/g, "$1").replace(/[\r\t\n]/g, " ");
+    }
+
+    drT.template = function(tmpl, c, def) {
+        c = c || drT.templateSettings;
+        var cse = c.append ? startend.append : startend.split, needhtmlencode, sid = 0, indv,
+            str  = (c.use || c.define) ? resolveDefs(c, tmpl, def || {}) : tmpl;
+
+        // console.log(str);
+        let beforeCode = '';
+        if(str.match(c.interpolate || skip)){
+            let inter_codes = str.match(c.interpolate || skip);
+            let inter_dict = {};
+            inter_codes.forEach(item=>{
+                item.replace(c.interpolate || skip,function (m,code) {
+                    let varname = code.split('.')[0];
+                    if(!inter_dict.hasOwnProperty(varname)){
+                        let beginCode = `if(typeof(${varname})==='undefined'){${varname}={}}`;
+                        inter_dict[varname] = beginCode;
+                    }if(!inter_dict.hasOwnProperty(code)){
+                        let beginCode = `if(typeof(${code})==='undefined'){${code}=''};`;
+                        inter_dict[code] = beginCode;
+                    }
+                });
+            });
+            let beginCode = Object.values(inter_dict).join('\n');
+            // console.log(beginCode);
+            beforeCode += beginCode;
+        }
+        str = beforeCode+("var out='" + (c.strip ? str.replace(/(^|\r|\n)\t* +| +\t*(\r|\n|$)/g," ")
+                .replace(/\r|\n|\t|\/\*[\s\S]*?\*\//g,""): str)
+                .replace(/'|\\/g, "\\$&")
+                .replace(c.encode || skip, function(m, code) {
+                    needhtmlencode = true;
+                    return cse.startencode + unescape(code) + cse.end;
+                })
+                .replace(c.interpolate || skip, function(m, code) {
+                    let varname = code.split('.')[0];
+                    // console.log(varname === code);
+                    // console.log(`varname:${varname},code:${code}`);
+                    if(varname === code){
+                        let res = cse.start + `JSON.stringify(${unescape(code)})` + cse.end;
+                        // console.log(res);
+                        return res
+                    }
+                    return cse.start + unescape(code) + cse.end;
+                })
+                .replace(c.conditional || skip, function(m, elsecase, code) {
+                    return elsecase ?
+                        (code ? "';}else if(" + unescape(code) + "){out+='" : "';}else{out+='") :
+                        (code ? "';if(" + unescape(code) + "){out+='" : "';}out+='");
+                })
+                .replace(c.iterate || skip, function(m, iterate, vname, iname) {
+                    if (!iterate) return "';} } out+='";
+                    sid+=1; indv=iname || "i"+sid; iterate=unescape(iterate);
+                    return "';var arr"+sid+"="+iterate+";if(arr"+sid+"){var "+vname+","+indv+"=-1,l"+sid+"=arr"+sid+".length-1;while("+indv+"<l"+sid+"){"
+                        +vname+"=arr"+sid+"["+indv+"+=1];out+='";
+                })
+                .replace(c.evaluate || skip, function(m, code) {
+                    return "';" + unescape(code) + "out+='";
+                })
+            + "';return out;")
+            .replace(/\n/g, "\\n").replace(/\t/g, '\\t').replace(/\r/g, "\\r")
+            .replace(/(\s|;|\}|^|\{)out\+='';/g, '$1').replace(/\+''/g, "");
+        //.replace(/(\s|;|\}|^|\{)out\+=''\+/g,'$1out+=');
+
+        if (needhtmlencode) {
+            // console.log('需要编码');
+            // console.log(c.doNotSkipEncoded);
+            if (!c.selfcontained && _globals && !_globals._encodeHTML) _globals._encodeHTML = drT.encodeHTMLSource(c.doNotSkipEncoded);
+            str = "var encodeHTML = typeof _encodeHTML !== 'undefined' ? _encodeHTML : ("
+                + drT.encodeHTMLSource.toString() + "(" + (c.doNotSkipEncoded || '') + "));"
+                + str;
+            // console.log(str);
+        }else{
+            // console.log('不需要编码');
+        }
+        // console.log(c.varname);
+        // console.log(str);
+        try {
+            return new Function(c.varname, str);
+        } catch (e) {
+            /* istanbul ignore else */
+            // console.log(e.message);
+            if (typeof console !== "undefined") console.log("Could not create a template function: " + str);
+            throw e;
+        }
+    };
+
+    drT.compile = function(tmpl, def) {
+        return drT.template(tmpl, null, def);
+    };
+    drT.renderText = function (tmpl,dict,varname){
+        varname = varname||'';
+        if(varname){
+            drT.templateSettings.varname = varname;
+        }
+        dict = dict||{};
+        return drT.compile(tmpl)(dict);
+    };
+}());

lib/drpy.ym.js

@@ -0,0 +1,4 @@
+import './util.ym.js'
+import dr from './drpy.min.js'
+ 
+__JS_SPIDER__ = dr.DRPY()

lib/util.ym.js

@@ -0,0 +1,158 @@
+import 'assets://js/lib/uri.min.js'
+import cheerio from 'assets://js/lib/cheerio.min.js';
+import 'assets://js/lib/crypto-js.js'
+
+var charStr = 'abacdefghjklmnopqrstuvwxyzABCDEFGHJKLMNOPQRSTUVWXYZ0123456789';
+export function randIndex(min, max, i) {
+    let index = Math.floor(Math.random() * (max - min + 1) + min),
+        numStart = charStr.length - 10;
+    if (i == 0 && index >= numStart) {
+        index = randIndex(min, max, i);
+    }
+    return index;
+}
+
+export function randomStr(len) {
+    let min = 0, max = charStr.length - 1, _str = '';
+    len = len || 15;
+    for (var i = 0, index; i < len; i++) {
+        index = randIndex(min, max, i);
+        _str += charStr[index];
+    }
+    return _str;
+}
+
+export function urljoin(base, url) {
+    base = base || '';
+    url = url || '';
+    let baseU = new Uri(base.trim().rstrip('/'));
+    url = url.trim().rstrip('/');
+    let u = undefined;
+    if (url.startsWith('http://') || url.startsWith('https://')) {
+        u = new Uri(url);
+    } else if (url.startsWith('://')) {
+        u = new Uri(baseU.protocol() + url);
+    } else if (url.startsWith('//')) {
+        u = new Uri(baseU.protocol() + ':' + url);
+    } else {
+        u = new Uri(baseU.protocol() + '://' + baseU.host() + (baseU.port() ? ':' + baseU.port() : '') + '/' + url);
+    }
+    if ((!u.path() || u.path().trim().length === 0) && baseU.path())
+        u.path(baseU.path());
+    if (!u.query() && baseU.query())
+        u.query(baseU.query());
+    return u.toString();
+}
+
+const DOM_CHECK_ATTR = /(url|src|href|data-original|data-src)$/;
+const SELECT_REGEX = /:eq|:lt|:gt|#/g;
+const SELECT_REGEX_A = /:eq|:lt|:gt/g;
+
+export function pdfh(html, parse, base_url) {
+    if (!parse || !parse.trim()) {
+        return ''
+    }
+    let eleFind = typeof html === 'object';
+    let option = undefined;
+    if (eleFind && parse.startsWith('body&&')) {
+        parse = parse.substr(6);
+        if (parse.indexOf('&&') < 0) {
+            option = parse.trim();
+            parse = '*=*';
+        }
+    }
+    if (parse.indexOf('&&') > -1) {
+        let sp = parse.split('&&');
+        option = sp[sp.length - 1];
+        sp.splice(sp.length - 1);
+        if (sp.length > 1) {
+            for (let i in sp) {
+                if (!SELECT_REGEX.test(sp[i])) {
+                    sp[i] = sp[i] + ':eq(0)';
+                }
+            }
+        } else {
+            if (!SELECT_REGEX.test(sp[0])) {
+                sp[0] = sp[0] + ':eq(0)';
+            }
+        }
+        parse = sp.join(' ');
+    }
+    let result = '';
+    const $ = eleFind ? html.rr : cheerio.load(html);
+    let ret = eleFind ? ((parse === '*=*' || $(html.ele).is(parse)) ? html.ele : $(html.ele).find(parse)) : $(parse);
+    if (option) {
+        if (option === 'Text') {
+            result = $(ret).text();
+        }
+        else if (option === 'Html') {
+            result = $(ret).html();
+        }
+        else {
+            result = $(ret).attr(option);
+        }
+        if (result && base_url && DOM_CHECK_ATTR.test(option)) {
+            if (/http/.test(result)) {
+                result = result.substr(result.indexOf('http'));
+            } else {
+                result = urljoin(base_url, result)
+            }
+        }
+    } else {
+        result = $(ret).toString();
+    }
+    return result;
+}
+
+export function pdfa(html, parse) {
+    if (!parse || !parse.trim()) {
+        return [];
+    }
+    let eleFind = typeof html === 'object';
+    if (parse.indexOf('&&') > -1) {
+        let sp = parse.split('&&');
+        for (let i in sp) {
+            if (!SELECT_REGEX_A.test(sp[i]) && i < sp.length - 1) {
+                sp[i] = sp[i] + ':eq(0)';
+            }
+        }
+        parse = sp.join(' ');
+    }
+    const $ = eleFind ? html.rr : cheerio.load(html);
+    let ret = eleFind ? ($(html.ele).is(parse) ? html.ele : $(html.ele).find(parse)) : $(parse);
+    let result = [];
+    if (ret) {
+        ret.each(function (idx, ele) {
+            result.push({ rr: $, ele: ele });
+        });
+    }
+    return result;
+}
+
+const defaultParser = {
+    pdfh:pdfh,
+    pdfa:pdfa,
+    pd(html,parse,uri){
+        let ret = this.pdfh(html,parse);
+        if(typeof(uri)==='undefined'||!uri){
+            uri = '';
+        }
+        if(DOM_CHECK_ATTR.test(parse)){
+            if(/http/.test(ret)){
+                ret = ret.substr(ret.indexOf('http'));
+            }else{
+                ret = urljoin(MY_URL,ret)
+            }
+        }
+        return ret
+    },
+};
+
+globalThis.randIndex = randIndex;
+globalThis.randomStr = randomStr;
+globalThis.urljoin = urljoin;
+globalThis.joinUrl = urljoin;
+globalThis.defaultParser = defaultParser;
+globalThis.pdfa = defaultParser.pdfa;
+globalThis.pdfh = defaultParser.pdfh;
+globalThis.pd = defaultParser.pd;