Browse Source

Merry Christmas 2020

pull/2/head
Perception 4 months ago
commit
1fb94dd192
14 changed files with 1232 additions and 0 deletions
  1. +36
    -0
      .eslintrc.js
  2. +6
    -0
      .gitignore
  3. +6
    -0
      .jshintrc
  4. +20
    -0
      .repl.js
  5. +20
    -0
      README.md
  6. +24
    -0
      bin/articles-update
  7. +7
    -0
      bin/download-all
  8. +2
    -0
      bin/queue
  9. +3
    -0
      bin/repl
  10. +37
    -0
      bin/scan
  11. +265
    -0
      index.js
  12. +747
    -0
      package-lock.json
  13. +24
    -0
      package.json
  14. +35
    -0
      sql/ds.sql

+ 36
- 0
.eslintrc.js View File

@ -0,0 +1,36 @@
module.exports = {
env: {
es6: true,
node: true,
},
extends: ['eslint:recommended'],
parserOptions: {
sourceType: 'module',
ecmaVersion: 9
},
plugins: [],
rules: {
'array-bracket-spacing': ['error', 'never'],
indent: ['warn', 2],
'linebreak-style': ['error', 'unix'],
'lines-between-class-members': ['warn', 'always'],
semi: ['error', 'always'],
'new-cap': 'off',
'no-console': 'off',
'no-debugger': 'off',
'no-mixed-spaces-and-tabs': 2,
'no-use-before-define': [2, 'nofunc'],
'no-unreachable': ['warn'],
'no-unused-vars': ['warn'],
'no-extra-parens': ['off'],
'no-mixed-operators': ['off'],
quotes: [2, 'single', 'avoid-escape'],
'block-scoped-var': 2,
'brace-style': [2, '1tbs', { allowSingleLine: true }],
'computed-property-spacing': [2, 'never'],
'keyword-spacing': 2,
'space-unary-ops': 2,
'max-len': ['warn', { 'code': 140 }]
}
};

+ 6
- 0
.gitignore View File

@ -0,0 +1,6 @@
node_modules
log
main.db
feed.xml
sections
*.swp

+ 6
- 0
.jshintrc View File

@ -0,0 +1,6 @@
{
"esversion": 8,
"asi": true,
"laxbreak": true,
"predef": [ "-Promise" ]
}

+ 20
- 0
.repl.js View File

@ -0,0 +1,20 @@
require('dotenv').config()
function reload(module){
delete require.cache[require.resolve(module)]
return require(module)
}
global.reload = reload
global.rl = reload
global.cl = console.log
global.axios = require('axios').default
global.cheerio = require('cheerio')
global.url = require('urlite')
global.RSSParser = require('rss-parser')
global.Database = require('better-sqlite3')
global.luxon = require('luxon')
global.DateTime = global.luxon.DateTime
global.ds = require('./index')
global.x = {}

+ 20
- 0
README.md View File

@ -0,0 +1,20 @@
# Daily Stormer Utilities
This is a little project to help me get out of my funk.
## Goals
* Crawl all dailystormer articles.
* Save crawl results as JSON files.
* Be able to run the crawler again to get new articles.
* This implies that I can keep track of what I've already crawled.
* Make it possible to do a full text search on those articles.
## Questions
### Crawling
The recursive crawl to get old articles and the crawl to update should be different. The updater
can assume a big crawl has already been done, and just pull articles down from the RSS feed. However,
the recursive crawl doesn't have that luxury, because it must find past articles and handle newly
discovered tags and categories as new crawl targets.

+ 24
- 0
bin/articles-update View File

@ -0,0 +1,24 @@
#!/usr/bin/env node
// Load RSS feed and insert new article data into the database
require('dotenv').config()
const program = require('commander')
const Database = require('better-sqlite3')
const Bluebird = require('bluebird')
const ds = require('../index')
const db = new Database('./main.db', {})
const feedURL = 'https://dailystormer.su/feed/'
async function main() {
const feed = await ds.getArticlesFromFeed(feedURL)
Bluebird.each(feed, (async (item) => {
const res = await ds.insertArticle(db, item)
if (res.success) {
console.warn(`+ "${item.title}"`)
} else {
console.warn(`= "${item.title}"`)
}
}))
}
main()

+ 7
- 0
bin/download-all View File

@ -0,0 +1,7 @@
#!/bin/bash
sections=(featured-stories world us society insight jewish-problem race-war)
for s in ${sections[@]} ; do
touch log/$s
bin/scan -t section $s
done

+ 2
- 0
bin/queue View File

@ -0,0 +1,2 @@
#!/usr/bin/env node
// Tail a file, and run bin/scan on new categories that have been appended.

+ 3
- 0
bin/repl View File

@ -0,0 +1,3 @@
#!/bin/sh
node -r ./.repl "$@"

+ 37
- 0
bin/scan View File

@ -0,0 +1,37 @@
#!/usr/bin/env node
// scan a category (or tag) for articles
require('dotenv').config()
const program = require('commander')
const Database = require('better-sqlite3')
const Bluebird = require('bluebird')
const ds = require('../index')
const db = new Database('./main.db', {})
const baseURL = 'https://dailystormer.su'
async function main() {
program.option('-t, --taxonomy <TYPE>', `"section" or "tag"`, 'section')
program.parse(process.argv)
//console.log(program)
const category = program.args[0]
if (!category) {
console.warn('category required')
process.exit(1)
}
const taxonomyBaseURL = `${baseURL}/${program.taxonomy}`
const partialArticles = await ds.scanCategory(taxonomyBaseURL, category)
const ps = partialArticles // partialArticles.slice(0, 3)
//console.log(ps)
Bluebird.each(ps, (async (art, i, length) => {
const more = await ds.getArticle(art.link)
const article = Object.assign({}, art, more)
const res = await ds.insertArticle(db, article)
if (res.success) {
console.log(`+ [${i+1}/${length}] "${art.title}"`)
} else {
console.log(`= [${i+1}/${length}] "${art.title}"`)
}
}))
}
main()

+ 265
- 0
index.js View File

@ -0,0 +1,265 @@
const axios = require('axios').default
const cheerio = require('cheerio')
const flatten = require('lodash.flatten')
const RSSParser = require('rss-parser')
const Bluebird = require('bluebird')
const url = require('urlite')
const luxon = require('luxon')
const DateTime = luxon.DateTime
const parser = new RSSParser()
async function getFeed(feedURL='https://dailystormer.su/feed/') {
const res = await axios.get(feedURL)
const feed = await parser.parseString(res.data)
return feed
}
/**
* Fetch the RSS feed and return its parsed contents.
* @param {String} feedURL - URL of the feed
* @returns {Object} parsed RSS feed
*/
async function getArticlesFromFeed(feedURL='https://dailystormer.su/feed/') {
const feed = await getFeed(feedURL);
return feed.items.map((a) => {
const slug = url.parse(a.link).pathname
const article = {
slug,
title: a.title,
author: a.creator,
content: a['content:encoded'],
categories: a.categories.map((cat) => cat.toLowerCase()),
tags: [],
published_date: a.isoDate,
//published_date: DateTime.fromRFC2822(a.pubDate).toISO()
}
return article
})
}
async function scanCategory(baseURL, category, options={}) {
const categoryURL = `${baseURL}/${category}`
console.log(categoryURL)
// scan first page of category to find out how many pages we have.
const res = await axios.get(categoryURL)
const $ = cheerio.load(res.data)
const p = parseInt($('.pagination .pages').text().split(/\s/)[3])
const pages = []
for (let i = 1; i <= p; i++) {
pages.push(i)
}
const scans = await Bluebird.map(pages, (n) => { return scanCategoryPage(baseURL, category, n) }, { concurrency: 4 })
const flatScans = flatten(scans)
return flatScans
}
async function scanCategoryPage(baseURL, category, page) {
var categoryURL
if (page === 1) {
categoryURL = `${baseURL}/${category}`
} else {
categoryURL = `${baseURL}/${category}/page/${page}`
}
const res = await axios.get(categoryURL)
console.log(categoryURL)
const $ = cheerio.load(res.data)
const articles = $('article.item-list').map((i, article) => {
const h2_a = $(article).find('h2.post-box-title a')
const link = h2_a.attr('href')
const slug = url.parse(link).pathname
const title = h2_a.text().trim()
const author = $(article).find('p.post-meta .post-meta-author').text().trim()
const publishedDateText = $(article).find('p.post-meta .tie-date').text().trim()
const publishedDate = DateTime.fromFormat(publishedDateText, 'LLLL d, yyyy', { zone: 'UTC' }).toISO()
return {
link,
slug,
title,
author,
published_date: publishedDate
}
}).toArray()
return articles
}
function unhyphen(word) {
const ws = word.split(/-/)
return ws.slice(1).map((s) => s.toLowerCase()).join(" ")
}
function extractTaxonomyFromArticle($article, type) {
const cs = $article.attr('class')
.split(/\s+/)
.filter((word) => word.match(new RegExp(`^${type}`)))
const ts = cs.map(unhyphen)
return ts
}
async function getArticle(url) {
const res = await axios.get(url)
const $ = cheerio.load(res.data)
const $article = $('article')
// get tags
const tags = extractTaxonomyFromArticle($article, 'tag')
// get categories
const categories = extractTaxonomyFromArticle($article, 'category')
// get content
const content = $article.find('div.entry').html()
return {
tags,
categories,
content
}
}
async function getCategory(db, name) {
const category = await db.prepare('SELECT id, name FROM category WHERE name = ? LIMIT 1').get(name)
return category
}
async function assocCategory(db, articleId, name) {
try {
var categoryId
const category = await getCategory(db, name)
if (!category) {
const res = await insertCategory(db, name)
categoryId = res.lastInsertRowid
} else {
categoryId = category.id
}
const res2 = await db.prepare('INSERT INTO article__category (article_id, category_id) VALUES (?, ?)')
.run(articleId, categoryId)
} catch (e) {
return { success: false, error: e }
}
return { success: true }
}
async function getTag(db, name) {
const tag = await db.prepare('SELECT id, name FROM tag WHERE name = ? LIMIT 1').get(name)
return tag
}
async function assocTag(db, articleId, name) {
try {
var tagId
const tag = await getTag(db, name)
if (!tag) {
const res = await insertTag(db, name)
tagId = res.lastInsertRowid
} else {
tagId = tag.id
}
const res2 = await db.prepare('INSERT INTO article__tag (article_id, tag_id) VALUES (?, ?)')
.run(articleId, tagId)
} catch (e) {
return { success: false, error: e }
}
return { success: true }
}
async function insertCategory(db, name) {
try {
const res = db.prepare(`INSERT INTO category (name) VALUES (?)`).run(name) // not async apparently
res.success = true
return res
} catch (e) {
return {
success: false,
error: e
}
}
}
async function insertTag(db, name) {
try {
const res = db.prepare(`INSERT INTO tag (name) VALUES (?)`).run(name) // not async apparently
res.success = true
return res
} catch (e) {
return {
success: false,
error: e
}
}
}
async function insertArticle(db, article) {
try {
// TODO wrap in a transaction?
// insert into article
const insert = db.prepare('INSERT INTO article (slug, title, author, content, published_date) VALUES (@slug, @title, @author, @content, @published_date)')
const res = await insert.run(article)
const articleId = res.lastInsertRowid
// insert into article_search for full-text search
const insertArticleSearch = db.prepare('INSERT INTO article_search (author, title, content, slug) VALUES (@author, @title, @content, @slug)')
const articleForSearch = Object.assign({}, article)
articleForSearch.content = cheerio.load(article.content).text()
await insertArticleSearch.run(articleForSearch)
// associate categories
article.categories.forEach((async (name) => {
await assocCategory(db, articleId, name)
}))
// associate tags
if (article.tags) {
article.tags.forEach((async (name) => {
await assocTag(db, articleId, name)
}))
}
}
catch (e) {
return { success: false, error: e }
}
return { success: true }
}
/**
* Perform a full-text search of the db.
* @param {Database} db - articles database
* @param {String} query - search query in SQLite3 fts5 syntax
*/
async function search(db, query, limit, offset) {
const stmt = db.prepare(`
SELECT rank, author, title, slug,
snippet(article_search, 2, '<b>', '</b>', '...', 32) as snippet
FROM article_search
WHERE article_search MATCH @query
ORDER BY rank
LIMIT @limit OFFSET @offset
`)
const res = stmt.all({ query, limit, offset })
return res
}
async function searchCount(db, query) {
const stmt = db.prepare(`SELECT count(*) as c FROM article_search WHERE article_search MATCH @query`)
const res = stmt.get({ query })
return res.c
}
// TODO write a function to insert an article into the database
// TODO write a function to insert a category into the database
// TODO write a function to associate an article with categories in a database
// TODO write a function to scan a category for articles
// TODO write a function to scrape an article into an insertable object
module.exports = {
getFeed,
getArticlesFromFeed,
scanCategory,
scanCategoryPage,
unhyphen,
extractTaxonomyFromArticle,
getArticle,
getCategory,
assocCategory,
getTag,
assocTag,
insertCategory,
insertTag,
insertArticle,
search,
searchCount
}

+ 747
- 0
package-lock.json View File

@ -0,0 +1,747 @@
{
"name": "ds",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@types/node": {
"version": "14.14.13",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.13.tgz",
"integrity": "sha512-vbxr0VZ8exFMMAjCW8rJwaya0dMCDyYW2ZRdTyjtrCvJoENMpdUHOT/eTzvgyA5ZnqRZ/sI0NwqAxNHKYokLJQ=="
},
"ansi-regex": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
},
"aproba": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
"integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
},
"are-we-there-yet": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
"integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
"requires": {
"delegates": "^1.0.0",
"readable-stream": "^2.0.6"
},
"dependencies": {
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"atomic-sleep": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
"integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="
},
"axios": {
"version": "0.21.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz",
"integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==",
"requires": {
"follow-redirects": "^1.10.0"
}
},
"base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
},
"better-sqlite3": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.1.1.tgz",
"integrity": "sha512-AkvGGyhAVZhRBOul2WT+5CB2EuveM3ZkebEKe1wxMqDZUy1XB/1RBgM66t0ybHC4DIni8+pr7NaLqEX87NUTwg==",
"requires": {
"bindings": "^1.5.0",
"prebuild-install": "^5.3.3",
"tar": "4.4.10"
}
},
"bindings": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
"integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
"requires": {
"file-uri-to-path": "1.0.0"
}
},
"bl": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
"integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
"requires": {
"buffer": "^5.5.0",
"inherits": "^2.0.4",
"readable-stream": "^3.4.0"
}
},
"bluebird": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
},
"boolbase": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
},
"buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"requires": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"cheerio": {
"version": "1.0.0-rc.3",
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
"integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
"requires": {
"css-select": "~1.2.0",
"dom-serializer": "~0.1.1",
"entities": "~1.1.1",
"htmlparser2": "^3.9.1",
"lodash": "^4.15.0",
"parse5": "^3.0.1"
}
},
"chownr": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
},
"code-point-at": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
"integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
},
"commander": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz",
"integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA=="
},
"console-control-strings": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
},
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"css-select": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
"integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
"requires": {
"boolbase": "~1.0.0",
"css-what": "2.1",
"domutils": "1.5.1",
"nth-check": "~1.0.1"
}
},
"css-what": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
"integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
},
"decompress-response": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz",
"integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==",
"requires": {
"mimic-response": "^2.0.0"
}
},
"deep-extend": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
},
"delegates": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
},
"detect-libc": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
"integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
},
"dom-serializer": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
"integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
"requires": {
"domelementtype": "^1.3.0",
"entities": "^1.1.1"
}
},
"domelementtype": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
"integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
},
"domhandler": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
"integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
"requires": {
"domelementtype": "1"
}
},
"domutils": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
"integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
"requires": {
"dom-serializer": "0",
"domelementtype": "1"
}
},
"dotenv": {
"version": "8.2.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
"integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw=="
},
"end-of-stream": {
"version": "1.4.4",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
"integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
"requires": {
"once": "^1.4.0"
}
},
"entities": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
},
"expand-template": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
"integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg=="
},
"fast-redact": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.0.0.tgz",
"integrity": "sha512-a/S/Hp6aoIjx7EmugtzLqXmcNsyFszqbt6qQ99BdG61QjBZF6shNis0BYR6TsZOQ1twYc0FN2Xdhwwbv6+KD0w=="
},
"fast-safe-stringify": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz",
"integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA=="
},
"file-uri-to-path": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
},
"flatstr": {
"version": "1.0.12",
"resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz",
"integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw=="
},
"follow-redirects": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.1.tgz",
"integrity": "sha512-SSG5xmZh1mkPGyKzjZP8zLjltIfpW32Y5QpdNJyjcfGxK3qo3NDDkZOZSFiGn1A6SclQxY9GzEwAHQ3dmYRWpg=="
},
"fs-constants": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
},
"fs-minipass": {
"version": "1.2.7",
"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz",
"integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==",
"requires": {
"minipass": "^2.6.0"
}
},
"gauge": {
"version": "2.7.4",
"resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
"integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
"requires": {
"aproba": "^1.0.3",
"console-control-strings": "^1.0.0",
"has-unicode": "^2.0.0",
"object-assign": "^4.1.0",
"signal-exit": "^3.0.0",
"string-width": "^1.0.1",
"strip-ansi": "^3.0.1",
"wide-align": "^1.1.0"
}
},
"github-from-package": {
"version": "0.0.0",
"resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
"integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4="
},
"has-unicode": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
"integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
},
"htmlparser2": {
"version": "3.10.1",
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
"integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
"requires": {
"domelementtype": "^1.3.1",
"domhandler": "^2.3.0",
"domutils": "^1.5.1",
"entities": "^1.1.1",
"inherits": "^2.0.1",
"readable-stream": "^3.1.1"
}
},
"ieee754": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
},
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"ini": {
"version": "1.3.8",
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
},
"is-fullwidth-code-point": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
"integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
"requires": {
"number-is-nan": "^1.0.0"
}
},
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"lodash": {
"version": "4.17.20",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
},
"lodash.flatten": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
"integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8="
},
"luxon": {
"version": "1.25.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.25.0.tgz",
"integrity": "sha512-hEgLurSH8kQRjY6i4YLey+mcKVAWXbDNlZRmM6AgWDJ1cY3atl8Ztf5wEY7VBReFbmGnwQPz7KYJblL8B2k0jQ=="
},
"mimic-response": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
"integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA=="
},
"minimist": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
},
"minipass": {
"version": "2.9.0",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz",
"integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==",
"requires": {
"safe-buffer": "^5.1.2",
"yallist": "^3.0.0"
}
},
"minizlib": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz",
"integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==",
"requires": {
"minipass": "^2.9.0"
}
},
"mkdirp": {
"version": "0.5.5",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
"integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
"requires": {
"minimist": "^1.2.5"
}
},
"mkdirp-classic": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
},
"napi-build-utils": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
"integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
},
"node-abi": {
"version": "2.19.3",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.19.3.tgz",
"integrity": "sha512-9xZrlyfvKhWme2EXFKQhZRp1yNWT/uI1luYPr3sFl+H4keYY4xR+1jO7mvTTijIsHf1M+QDe9uWuKeEpLInIlg==",
"requires": {
"semver": "^5.4.1"
}
},
"noop-logger": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz",
"integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI="
},
"npmlog": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
"integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
"requires": {
"are-we-there-yet": "~1.1.2",
"console-control-strings": "~1.1.0",
"gauge": "~2.7.3",
"set-blocking": "~2.0.0"
}
},
"nth-check": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
"integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
"requires": {
"boolbase": "~1.0.0"
}
},
"number-is-nan": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
"integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
},
"object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"requires": {
"wrappy": "1"
}
},
"parse5": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
"integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
"requires": {
"@types/node": "*"
}
},
"pino": {
"version": "6.8.0",
"resolved": "https://registry.npmjs.org/pino/-/pino-6.8.0.tgz",
"integrity": "sha512-nxq+6Jr7m0cMjYFBoTRw3bco14omZ/SQCheAHz9GVwdkbUrzKhgT+gSI/ql2Mnsca0QQKgpB/ACWhjxE4JsX3Q==",
"requires": {
"fast-redact": "^3.0.0",
"fast-safe-stringify": "^2.0.7",
"flatstr": "^1.0.12",
"pino-std-serializers": "^2.4.2",
"quick-format-unescaped": "^4.0.1",
"sonic-boom": "^1.0.2"
}
},
"pino-std-serializers": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-2.5.0.tgz",
"integrity": "sha512-wXqbqSrIhE58TdrxxlfLwU9eDhrzppQDvGhBEr1gYbzzM4KKo3Y63gSjiDXRKLVS2UOXdPNR2v+KnQgNrs+xUg=="
},
"prebuild-install": {
"version": "5.3.6",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.6.tgz",
"integrity": "sha512-s8Aai8++QQGi4sSbs/M1Qku62PFK49Jm1CbgXklGz4nmHveDq0wzJkg7Na5QbnO1uNH8K7iqx2EQ/mV0MZEmOg==",
"requires": {
"detect-libc": "^1.0.3",
"expand-template": "^2.0.3",
"github-from-package": "0.0.0",
"minimist": "^1.2.3",
"mkdirp-classic": "^0.5.3",
"napi-build-utils": "^1.0.1",
"node-abi": "^2.7.0",
"noop-logger": "^0.1.1",
"npmlog": "^4.0.1",
"pump": "^3.0.0",
"rc": "^1.2.7",
"simple-get": "^3.0.3",
"tar-fs": "^2.0.0",
"tunnel-agent": "^0.6.0",
"which-pm-runs": "^1.0.0"
}
},
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"pump": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
"integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
}
},
"quick-format-unescaped": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz",
"integrity": "sha512-RyYpQ6Q5/drsJyOhrWHYMWTedvjTIat+FTwv0K4yoUxzvekw2aRHMQJLlnvt8UantkZg2++bEzD9EdxXqkWf4A=="
},
"rc": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
"integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
"requires": {
"deep-extend": "^0.6.0",
"ini": "~1.3.0",
"minimist": "^1.2.0",
"strip-json-comments": "~2.0.1"
}
},
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
},
"rss-parser": {
"version": "3.10.0",
"resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.10.0.tgz",
"integrity": "sha512-TC6FNvEmdFeaW6r/60MSJT7cp4d95X4M9As+mvNtxRx7YXHxpV95syMnWZthZSeD1BRN7SEKdq6c3nxMLQRopw==",
"requires": {
"entities": "^2.0.3",
"xml2js": "^0.4.19"
},
"dependencies": {
"entities": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz",
"integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w=="
}
}
},
"safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
},
"sax": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
},
"semver": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
"integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
},
"set-blocking": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
"integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
},
"signal-exit": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz",
"integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA=="
},
"simple-concat": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
"integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q=="
},
"simple-get": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz",
"integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==",
"requires": {
"decompress-response": "^4.2.0",
"once": "^1.3.1",
"simple-concat": "^1.0.0"
}
},
"sonic-boom": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.3.0.tgz",
"integrity": "sha512-4nX6OYvOYr6R76xfQKi6cZpTO3YSWe/vd+QdIfoH0lBy0MnPkeAbb2rRWgmgADkXUeCKPwO1FZAKlAVWAadELw==",
"requires": {
"atomic-sleep": "^1.0.0",
"flatstr": "^1.0.12"
}
},
"string-width": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
"integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
"strip-ansi": "^3.0.0"
}
},
"string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
"requires": {
"safe-buffer": "~5.2.0"
}
},
"strip-ansi": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
"requires": {
"ansi-regex": "^2.0.0"
}
},
"strip-json-comments": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
"integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
},
"tar": {
"version": "4.4.10",
"resolved": "https://registry.npmjs.org/tar/-/tar-4.4.10.tgz",
"integrity": "sha512-g2SVs5QIxvo6OLp0GudTqEf05maawKUxXru104iaayWA09551tFCTI8f1Asb4lPfkBr91k07iL4c11XO3/b0tA==",
"requires": {
"chownr": "^1.1.1",
"fs-minipass": "^1.2.5",
"minipass": "^2.3.5",
"minizlib": "^1.2.1",
"mkdirp": "^0.5.0",
"safe-buffer": "^5.1.2",
"yallist": "^3.0.3"
}
},
"tar-fs": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
"integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
"requires": {
"chownr": "^1.1.1",
"mkdirp-classic": "^0.5.2",
"pump": "^3.0.0",
"tar-stream": "^2.1.4"
}
},
"tar-stream": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz",
"integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==",
"requires": {
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
}
},
"tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"requires": {
"safe-buffer": "^5.0.1"
}
},
"urlite": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/urlite/-/urlite-3.0.0.tgz",
"integrity": "sha512-Ffgts2pTfFFybsHm1uk8s01jBpIBIU8/ZBrTVFpsM5eA1AYVsEfBYkiSahyJcuCEdLvfOkjq5vXznlcI1HhfRw=="
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"which-pm-runs": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz",
"integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs="
},
"wide-align": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
"integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
"requires": {
"string-width": "^1.0.2 || 2"
}
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"xml2js": {
"version": "0.4.23",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
"integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
"requires": {
"sax": ">=0.6.0",
"xmlbuilder": "~11.0.0"
}
},
"xmlbuilder": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
},
"yallist": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
}
}
}

+ 24
- 0
package.json View File

@ -0,0 +1,24 @@
{
"name": "ds",
"version": "1.0.0",
"description": "Daily Stormer Utilities",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "MIT",
"dependencies": {
"axios": "^0.21.0",
"better-sqlite3": "^7.1.1",
"bluebird": "^3.7.2",
"cheerio": "^1.0.0-rc.3",
"commander": "^6.2.1",
"dotenv": "^8.2.0",
"lodash.flatten": "^4.4.0",
"luxon": "^1.25.0",
"pino": "^6.8.0",
"rss-parser": "^3.10.0",
"urlite": "^3.0.0"
}
}

+ 35
- 0
sql/ds.sql View File

@ -0,0 +1,35 @@
CREATE VIRTUAL TABLE article_search USING fts5(author, title, content, slug);
CREATE TABLE article (
id INTEGER PRIMARY KEY AUTOINCREMENT,
slug VARCHAR(256) NOT NULL,
title VARCHAR(256) NOT NULL,
author VARCHAR(256) NOT NULL,
content text,
published_date DATETIME NOT NULL,
UNIQUE(slug)
);
CREATE TABLE category (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name VARCHAR(128) NOT NULL,
UNIQUE(name)
);
CREATE TABLE tag (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name VARCHAR(128) NOT NULL,
UNIQUE(name)
);
CREATE TABLE article__category (
article_id INTEGER,
category_id INTEGER,
UNIQUE(category_id, article_id)
);
CREATE TABLE article__tag (
article_id INTEGER,
tag_id INTEGER,
UNIQUE(tag_id, article_id)
);

Loading…
Cancel
Save