Add Twitter integration

Add Job Listening Integration
Various Refactorings

src/main/kotlin/de/wulkanat/AdminCli.kt

@@ -1,8 +1,7 @@
 package de.wulkanat
 
-import de.wulkanat.model.BlogPostPreview
+import de.wulkanat.web.fakeUpdateBlogPost
 import net.dv8tion.jda.api.hooks.ListenerAdapter
-import de.wulkanat.web.SiteWatcher
 import net.dv8tion.jda.api.EmbedBuilder
 import net.dv8tion.jda.api.events.message.priv.PrivateMessageReceivedEvent
 import org.hmcore.TwitterJob
@@ -24,14 +23,9 @@ class AdminCli : ListenerAdapter() {
         when (command[0].value) {
             "stop" -> exitProcess(1)
             "fakeUpdate" -> {
-                SiteWatcher.newestBlog = BlogPostPreview(
-                    title = "FakePost",
-                    imgUrl = "",
-                    fullPostUrl = "",
-                    author = "wulkanat",
-                    date = "now",
-                    description = "Lorem Ipsum"
-                )
+                // TODO: implement fake update for blog posts
+                // BLOG_POST_WATCHER.current = setOf()
+                fakeUpdateBlogPost()
 
                 TwitterJob.lastTweetID = "poggers"
 

src/main/kotlin/de/wulkanat/DataIO.kt

@@ -2,7 +2,11 @@
 package de.wulkanat
 
 import de.wulkanat.extensions.ensureExists
+import de.wulkanat.model.BlogPostPreview
+import de.wulkanat.model.JobListingPreview
+import kotlinx.serialization.Required
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import java.io.File
@@ -28,23 +32,32 @@ data class CustomMessage(
 
 @Serializable
 data class AdminFile(
-    val adminId: Long = 12345,
-    val token: String = "12345",
-    val updateMs: Long = 30000,
-    val shards: Int = 6,
-    val watchingMessage: String = "for new Blogposts",
-    val offlineMessage: String = "CONNECTION FAILED",
-    var twitterApi: TwitterApi? = TwitterApi()
+    @Required val adminId: Long = 12345,
+    @Required val token: String = "12345",
+    @Required val updateMs: Long = 30000,
+    @Required val shards: Int = 6,
+    @Required val watchingMessage: String = "for new Blogposts",
+    @Required val offlineMessage: String = "CONNECTION FAILED",
+    @Required var twitterApi: TwitterApi? = TwitterApi()
 )
 
 @Serializable
 data class TwitterApi(
-    val accessToken: String = "accessTokenHere",
-    val accessTokenSecret: String = "accessTokenSecretHere",
-    val apiKey: String = "apiKeyHere",
-    val apiKeySecret: String = "Api Key secret here"
+    @Required val accessToken: String = "accessTokenHere",
+    @Required val accessTokenSecret: String = "accessTokenSecretHere",
+    @Required val apiKey: String = "apiKeyHere",
+    @Required val apiKeySecret: String = "Api Key secret here"
 )
 
+@Serializable
+data class Webhooks(
+    @Required val blogPostsWebhookUrl: String = "https://...",
+    @Required val jobListingsWebhookUrl: String = "https://...",
+)
+
+val WEBHOOKS_FILE = File("webhooks.json").ensureExists(Json.encodeToString(Webhooks()))
+val WEBHOOKS = Json.decodeFromString<Webhooks>(WEBHOOKS_FILE.readText())
+
 val SERVERS_FILE = File("servers.json").ensureExists(Json.encodeToString(listOf<DiscordChannel>()))
 val SERVICE_CHANNELS_FILE =
     File("service_channels.json").ensureExists(Json.encodeToString(listOf<ServiceChannel>()))

src/main/kotlin/de/wulkanat/Main.kt

@@ -1,6 +1,6 @@
 package de.wulkanat
 
-import de.wulkanat.web.SiteWatcher
+import de.wulkanat.web.getNewBlogPosts
 import net.dv8tion.jda.api.JDA
 import net.dv8tion.jda.api.JDABuilder
 import net.dv8tion.jda.api.MessageBuilder
@@ -59,8 +59,8 @@ object Main {
         })
 
         timer("Updater", daemon = true, initialDelay = 0L, period = Admin.updateMs) {
-            if (SiteWatcher.hasNewBlogPost()) {
-                Channels.sentToAll(MessageBuilder().setEmbed(SiteWatcher.newestBlog!!.toMessageEmbed()).build())
+            getNewBlogPosts()?.forEach {
+                Channels.sentToAll(MessageBuilder().setEmbed(it.toMessageEmbed()).build())
             }
         }
 

src/main/kotlin/de/wulkanat/extensions/Jsoup.kt

@@ -7,4 +7,5 @@ operator fun Element.get(className: String): Elements =
     this.getElementsByClass(className)
 
 val Elements.text get() = text().trim()
-val Element.absUrl get(): String = child(0).absUrl("href")
+val Element.absUrl get(): String = child(0).absUrl("href")
+val Element.imgSrc get(): String = child(0).attr("src")

src/main/kotlin/de/wulkanat/model/BlogPostPreview.kt

@@ -3,7 +3,9 @@ package de.wulkanat.model
 import net.dv8tion.jda.api.EmbedBuilder
 import net.dv8tion.jda.api.entities.MessageEmbed
 import de.wulkanat.extensions.hex2Rgb
+import kotlinx.serialization.Serializable
 
+@Serializable
 data class BlogPostPreview(
     val title: String,
     val description: String,

src/main/kotlin/de/wulkanat/model/JobListingPreview.kt

@@ -1,10 +1,12 @@
 package de.wulkanat.model
 
 import de.wulkanat.extensions.hex2Rgb
+import kotlinx.serialization.Serializable
 import net.dv8tion.jda.api.EmbedBuilder
 import net.dv8tion.jda.api.entities.MessageEmbed
 
-class JobListingPreview(
+@Serializable
+data class JobListingPreview(
     val title: String,
     val department: String,
     val location: String,

src/main/kotlin/de/wulkanat/web/BlogPostParser.kt

@@ -1,24 +0,0 @@
-package de.wulkanat.web
-
-import de.wulkanat.model.BlogPostPreview
-import org.jsoup.nodes.Document
-import org.jsoup.nodes.Element
-
-object BlogPostParser {
-    fun getFistBlog(doc: Document): BlogPostPreview {
-        val posts = doc.getElementsByClass("postWrapper")
-        return parseBlog(posts.first())
-    }
-
-    private fun parseBlog(elm: Element): BlogPostPreview {
-
-        return BlogPostPreview(
-            title = elm.getElementsByClass("post__details__heading").first().text(),
-            imgUrl = elm.getElementsByClass("post__image__frame").first().child(0).attr("src"),
-            fullPostUrl = elm.child(0).absUrl("href"),
-            date = elm.getElementsByClass("post__details__meta__date").first().text(),
-            author = elm.getElementsByClass("post__details__meta__author").first().text(),
-            description = elm.getElementsByClass("post__details__body").first().text()
-        )
-    }
-}

src/main/kotlin/de/wulkanat/web/JobListingParser.kt

@@ -1,21 +0,0 @@
-package de.wulkanat.web
-
-import de.wulkanat.extensions.get
-import de.wulkanat.extensions.text
-import de.wulkanat.extensions.absUrl
-import de.wulkanat.model.JobListingPreview
-import org.jsoup.nodes.Document
-
-fun parseJobListings(doc: Document) =
-    doc["current-jobs__departments"].flatMap { jobDepartment ->
-        val jobDepartmentName = jobDepartment["current-jobs__department-name"].text
-
-        jobDepartment["current-jobs__job"].map { job ->
-            JobListingPreview(
-                title = job["current-jobs__job-title"].text,
-                department = jobDepartmentName,
-                location = job["current-jobs__job-location"].text,
-                fullListingUrl = job.absUrl
-            )
-        }
-    }

src/main/kotlin/de/wulkanat/web/Parser.kt

@@ -0,0 +1,40 @@
+package de.wulkanat.web
+
+import de.wulkanat.extensions.absUrl
+import de.wulkanat.extensions.get
+import de.wulkanat.extensions.imgSrc
+import de.wulkanat.extensions.text
+import de.wulkanat.model.BlogPostPreview
+import de.wulkanat.model.JobListingPreview
+
+private const val BLOG_POST_STATE_FILE_NAME = "blog_state.json"
+fun fakeUpdateBlogPost() = removeFirstFromSiteSave<BlogPostPreview>(BLOG_POST_STATE_FILE_NAME)
+fun getNewBlogPosts() = updateSite("https://hytale.com/news", BLOG_POST_STATE_FILE_NAME) { doc ->
+    doc["postWrapper"].map {
+        BlogPostPreview(
+            title = it["post__details__heading"].text,
+            imgUrl = it["post__image__frame"].first().imgSrc,
+            fullPostUrl = it.absUrl,
+            date = it["post__details__meta__date"].text,
+            author = it["post__details__meta__author"].text,
+            description = it["post__details__body"].text,
+        )
+    }
+}
+
+private const val JOB_LISTING_STATE_FILE_NAME = "jobs_state.json"
+fun fakeUpdateJobListings() = removeFirstFromSiteSave<JobListingPreview>(JOB_LISTING_STATE_FILE_NAME)
+fun getNewJobListings() = updateSite("https://hypixelstudios.com/jobs/", JOB_LISTING_STATE_FILE_NAME) { doc ->
+    doc["current-jobs__departments"].flatMap { jobDepartment ->
+        val jobDepartmentName = jobDepartment["current-jobs__department-name"].text
+
+        jobDepartment["current-jobs__job"].map { job ->
+            JobListingPreview(
+                title = job["current-jobs__job-title"].text,
+                department = jobDepartmentName,
+                location = job["current-jobs__job-location"].text,
+                fullListingUrl = job.absUrl
+            )
+        }
+    }
+}

src/main/kotlin/de/wulkanat/web/SiteWatcher.kt

@@ -1,44 +1,37 @@
 package de.wulkanat.web
 
-import de.wulkanat.Admin
-import de.wulkanat.DiscordRpc
-import de.wulkanat.model.BlogPostPreview
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
 import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import java.io.File
 import java.io.IOException
 
-object SiteWatcher {
-    private const val BLOG_INDEX_URL = "https://www.hytale.com/news"
-    var newestBlog: BlogPostPreview? = null
-    private var siteOnline = false
+/**
+ * Removes the first element of a saved JSON list file
+ */
+inline fun <reified T>
+        removeFirstFromSiteSave(fileName: String) = File(fileName).takeIf { it.exists() }?.let {
+    it.writeText(Json.encodeToString(Json.decodeFromString<List<T>>(it.readText()).toMutableList().apply { removeFirst() }))
+}
 
-    fun hasNewBlogPost(): Boolean {
-        try {
-            val doc = Jsoup.connect(BLOG_INDEX_URL).get()
-            val newBlog = BlogPostParser.getFistBlog(doc)
+inline fun <reified T> updateSite(url: String, fileName: String, parser: (Document) -> List<T>) = try {
+    val currentStateFile = File(fileName)
 
-            if (newestBlog == newBlog) {
-                return false
-            }
+    val retrievedElements = parser(Jsoup.connect(url).get())
+    var currentElements = if (currentStateFile.exists())
+        Json.decodeFromString(currentStateFile.readText()) else retrievedElements
 
-            if (newestBlog == null) {
-                newestBlog = newBlog
-                return false
-            } else {
-                newestBlog = newBlog
-            }
-        } catch (e: IOException) {
-            Admin.error("Connection to Hytale Server failed", e.message ?: e.localizedMessage)
-            siteOnline = false
-            DiscordRpc.updatePresence(siteOnline)
+    val newElements = retrievedElements - currentElements
+    currentElements = retrievedElements
+    currentStateFile.writeText(Json.encodeToString(currentElements))
 
-            return false
-        }
+    newElements
+} catch (e: IOException) {
+    // TODO: put this somewhere else
+    // Admin.error("""Fetching "$url" failed!""", e.message ?: e.localizedMessage)
+    // DiscordRpc.updatePresence(canUpdate.also { canUpdate = false })
 
-        if (!siteOnline) {
-            siteOnline = true
-            DiscordRpc.updatePresence(siteOnline)
-        }
-
-        return true
-    }
-}
+    null
+}