Browse Source

Some quick cleanup/CL style guide adherence

master
Viola 7 months ago
parent
commit
82ef326993
3 changed files with 108 additions and 71 deletions
  1. +2
    -0
      .gitignore
  2. +78
    -71
      src/cl-kiwi.lisp
  3. +28
    -0
      t/cl-kiwi.lisp

+ 2
- 0
.gitignore View File

@@ -17,3 +17,5 @@
*.wx64fsl
*.wx32fsl

# Also ignore emacs backups
*~

cl-kiwi.lisp → src/cl-kiwi.lisp View File

@@ -1,9 +1,6 @@
#!/usr/bin/sbcl

;;;; Some stuff for playing around with KF thread metrics here.
;;;; Some Kiwi Farms analytics using Common Lisp!
;;;;
;;;; Viola, 01/01/2020
;;;; Licensed with Unlicense, etc.
;;;; (more to come here. Also for the markdown readme.)

;;; Load the necessary libraries (maybe packagedef at some point?)
;;; Also, might want to quickload these with quicklisp, in case they
@@ -32,75 +29,80 @@
;;; in the post object.
(defclass post ()
((author :reader get-post-author
:initarg :author
:type string
:documentation "The author of this post.")
:initarg :author
:type string
:documentation "The author of this post.")
(body :reader get-post-body
:initarg :body
:type string
:documentation "The HTML body of this post.")
:initarg :body
:type string
:documentation "The HTML body of this post.")
(edit-time :reader get-post-edit-time
:initarg :edit-time
:type string
:documentation "The time this post was edited (NIL if never).")
:initarg :edit-time
:type string
:documentation "The time this post was edited (NIL if never).")
(global-number :reader get-post-global-number
:initarg :global-number
:type integer
:documentation "The post number (with respect to the forum).")
:initarg :global-number
:type integer
:documentation "The post number (with respect to the forum).")
(number :reader get-post-number
:initarg :number
:type integer
:documentation "The post number (with respect to the thread).")
:initarg :number
:type integer
:documentation "The post number (with respect to the thread).")
(ratings :reader get-post-ratings
:initarg :ratings
:type property-list
:documentation "This post's ratings, as a plist.")
:initarg :ratings
:type property-list
:documentation "This post's ratings, as a plist.")
(time :reader get-post-time
:initarg :time
:type string
:documentation "The time this post was made.")
:initarg :time
:type string
:documentation "The time this post was made.")
(url :reader get-post-url
:initarg :url
:type string
:documentation "The URL for this post."))
:initarg :url
:type string
:documentation "The URL for this post."))
(:documentation "A post in a Kiwi Farms thread."))


(defmethod print-object ((obj post) stream)
"Pretty printing for post objects in the REPL."
(print-unreadable-object (obj stream :type t)
(format stream
"#~2,'0d: ~a, ~a"
(get-post-number obj)
(get-post-author obj)
(get-post-time obj))))
"#~2,'0d: ~a, ~a"
(get-post-number obj)
(get-post-author obj)
(get-post-time obj))))

(defmethod print-post-listing ((obj post) &optional body-p)
"Print a pretty post listing. If BODY-P, the post body is also printed."
(format t
"~a~%#~2,'0d (#~7,'0d): ~a~%Posted: ~a~%Edited: ~a~%Ratings: ~a~%"
(get-post-url obj)
(get-post-number obj)
(get-post-global-number obj)
(get-post-author obj)
(get-post-time obj)
(get-post-edit-time obj)
(get-post-ratings obj))
"~a~%#~2,'0d (#~7,'0d): ~a~%Posted: ~a~%Edited: ~a~%Ratings: ~a~%"
(get-post-url obj)
(get-post-number obj)
(get-post-global-number obj)
(get-post-author obj)
(get-post-time obj)
(get-post-edit-time obj)
(get-post-ratings obj))
(when body-p
(format t "~a~%" (get-post-body obj)))
(format t "~%"))

;;; TODO: possibly want a 'to-serialized' method that prints a readable sexpr
;;; form of the posts so we can potentially store them and not have to
;;; spend so much time re-downloading them each time?

;;; (cont.) if so, then this function should be 'build-post-from-dom',
;;; and have another function 'build-post-from-serialized'.
(defun build-post (dom-element)
"Build a post object given the (plump) DOM article element for the post."
(make-instance 'post
:author (extract-author dom-element)
:body (extract-body dom-element)
:edit-time (extract-edit-time dom-element)
:global-number (extract-global-number dom-element)
:number (extract-number dom-element)
:ratings (extract-ratings dom-element)
:time (extract-time dom-element)
:url (extract-url dom-element)))
:author (extract-author dom-element)
:body (extract-body dom-element)
:edit-time (extract-edit-time dom-element)
:global-number (extract-global-number dom-element)
:number (extract-number dom-element)
:ratings (extract-ratings dom-element)
:time (extract-time dom-element)
:url (extract-url dom-element)))

(defun extract-author (dom-element)
"Extract the post author text from HTML/DOM."
@@ -119,7 +121,7 @@
(defun extract-global-number (dom-element)
"Extract the forum-global post number from HTML/DOM."
(parse-integer (elt (lquery:$ dom-element (attr :data-content)) 0)
:start (length "post-")))
:start (length "post-")))

(defun extract-number (dom-element)
"Extract the post number from HTML/DOM."
@@ -130,23 +132,23 @@
(defun extract-ratings (dom-element)
"Extract the post ratings from HTML/DOM. (Requires another HTTP call.)"
(let* ((ratings-page (concatenate 'string
"https://kiwifarms.net/posts/"
(write-to-string
(extract-global-number dom-element))
"/reactions"))
(request (dex:get ratings-page))
(parsed (lquery:$ (initialize request)))
(ratings-html (lquery:$ parsed "bdi" (parent)))
(ratings-plist nil))
"https://kiwifarms.net/posts/"
(write-to-string
(extract-global-number dom-element))
"/reactions"))
(request (dex:get ratings-page))
(parsed (lquery:$ (initialize request)))
(ratings-html (lquery:$ parsed "bdi" (parent)))
(ratings-plist nil))
; We ignore the first element of ratings-html, it's the 'All' count
(loop for index from 1 below (length ratings-html) do
; We ignore the first element of ratings-html, it's the 'All' count
(let* ((span-element (elt ratings-html index))
(ratings-text (elt (lquery:$ span-element (text)) 0)))
(push (elt (lquery:$ span-element "bdi" (text)) 0) ratings-plist)
(push (parse-integer (subseq ratings-text
(1+ (search "(" ratings-text))
(1- (length ratings-text))))
ratings-plist))
(let* ((span-element (elt ratings-html index))
(ratings-text (elt (lquery:$ span-element (text)) 0)))
(push (elt (lquery:$ span-element "bdi" (text)) 0) ratings-plist)
(push (parse-integer (subseq ratings-text
(1+ (search "(" ratings-text))
(1- (length ratings-text))))
ratings-plist)))
(reverse ratings-plist)))
(defun extract-time (dom-element)
@@ -156,8 +158,13 @@
(defun extract-url (dom-element)
"Extract the post's URL from HTML/DOM."
(concatenate 'string
*thread-url*
(elt (lquery:$ dom-element (attr :data-content)) 0)))
*thread-url*
(elt (lquery:$ dom-element (attr :data-content)) 0)))



;;; Script stuff (maybe want this in a separate file/it's own function?)


;;; The URL for the Kiwi Farms thread (e.g.)
;;; (read this in from ARGV serialization at some point? TODO.)
@@ -189,8 +196,8 @@
;;; TODO: Can I write this better? It's kind of gross doing it this way..
(setf *page-posts*
(lquery:$ *page-posts*
(filter
(lambda (x) (elt (lquery:$ x (attr :data-author)) 0)))))
(filter
(lambda (x) (elt (lquery:$ x (attr :data-author)) 0)))))


;;; build all the posts in one go by mapping across them!

+ 28
- 0
t/cl-kiwi.lisp View File

@@ -0,0 +1,28 @@
;;;; Unit tests when I get around to writing them.
;;;; (using the FiveAM testing framework)
;;;; TODO: maybe also code coverage with sb-cover eventually?

(require 'fiveam)
(use-package 'fiveam)

(def-suite cl-kiwi-test-suite :description "Unit testing for cl-kiwi")
(in-suite cl-kiwi-test-suite)

;;; Define the tests
;;; TODO: sort out how I want to import the functions/class from
;;; ../t/cl-kiwi.
;;; do I want it all packaged+exported after all?
(test test-1
"Example test. They'll all pretty much look like this."
(is (= 4 (+ 2 2)) "2+2=4 test failed (using #'= to test equality)"))

(test test-2
"Also make sure the methods/functions error when they're supposed to:"
(signals
(error "Adding an integer to a symbol didn't signal an error!")
(+ 2 'symbol)))

;;; Run the tests!
(run!)



Loading…
Cancel
Save