From 37f42dc10718cd8f50ac705e8f33ca8dda2c4a3b Mon Sep 17 00:00:00 2001 From: Harsh Shandilya Date: Mon, 17 Jul 2023 13:36:42 +0530 Subject: [PATCH] feat(api): add a Retrofit service for search --- api/build.gradle.kts | 1 + .../msfjarvis/claw/api/LobstersSearchApi.kt | 20 +++++ .../claw/api/converters/SearchConverter.kt | 86 +++++++++++++++++++ .../dev/msfjarvis/claw/api/ApiWrapper.kt | 12 +-- .../dev/msfjarvis/claw/api/SearchApiTest.kt | 59 +++++++++++++ .../msfjarvis/claw/api/SearchApiWrapper.kt | 30 +++++++ .../dev/msfjarvis/claw/util/TestUtils.kt | 2 +- .../test/resources/search_chatgpt_page.html | 5 ++ 8 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 api/src/main/kotlin/dev/msfjarvis/claw/api/LobstersSearchApi.kt create mode 100644 api/src/main/kotlin/dev/msfjarvis/claw/api/converters/SearchConverter.kt create mode 100644 api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiTest.kt create mode 100644 api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiWrapper.kt create mode 100644 api/src/test/resources/search_chatgpt_page.html diff --git a/api/build.gradle.kts b/api/build.gradle.kts index 072c9819..2a5e3df1 100644 --- a/api/build.gradle.kts +++ b/api/build.gradle.kts @@ -24,6 +24,7 @@ dependencies { implementation(libs.dagger) implementation(libs.javax.inject) + implementation(libs.jsoup) testImplementation(testFixtures(libs.eithernet)) testImplementation(libs.kotlinx.coroutines.test) diff --git a/api/src/main/kotlin/dev/msfjarvis/claw/api/LobstersSearchApi.kt b/api/src/main/kotlin/dev/msfjarvis/claw/api/LobstersSearchApi.kt new file mode 100644 index 00000000..145565b1 --- /dev/null +++ b/api/src/main/kotlin/dev/msfjarvis/claw/api/LobstersSearchApi.kt @@ -0,0 +1,20 @@ +/* + * Copyright © 2023 Harsh Shandilya. + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. + */ +package dev.msfjarvis.claw.api + +import com.slack.eithernet.ApiResult +import dev.msfjarvis.claw.model.LobstersPost +import retrofit2.http.GET +import retrofit2.http.Query + +interface LobstersSearchApi { + @GET("/search?q={query}&what=stories&order=newest&page={page}") + suspend fun searchPosts( + @Query("query") searchQuery: String, + @Query("page") page: Int, + ): ApiResult, Unit> +} diff --git a/api/src/main/kotlin/dev/msfjarvis/claw/api/converters/SearchConverter.kt b/api/src/main/kotlin/dev/msfjarvis/claw/api/converters/SearchConverter.kt new file mode 100644 index 00000000..1faf42a9 --- /dev/null +++ b/api/src/main/kotlin/dev/msfjarvis/claw/api/converters/SearchConverter.kt @@ -0,0 +1,86 @@ +/* + * Copyright © 2023 Harsh Shandilya. + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. + */ +package dev.msfjarvis.claw.api.converters + +import dev.msfjarvis.claw.api.LobstersApi +import dev.msfjarvis.claw.model.LobstersPost +import dev.msfjarvis.claw.model.User +import java.lang.reflect.Type +import javax.inject.Inject +import okhttp3.ResponseBody +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import org.jsoup.select.Elements +import retrofit2.Converter +import retrofit2.Retrofit + +class SearchConverter @Inject constructor() : Converter> { + override fun convert(value: ResponseBody): List { + val elements = + Jsoup.parse(value.string(), LobstersApi.BASE_URL).select("div.story_liner.h-entry") + return elements.map(::parsePost) + } + + private fun parsePost(elem: Element): LobstersPost { + val parent = elem.parent() ?: error("$elem must have a parent") + val shortId = parent.attr("data-shortid") + val titleElement = elem.select("span.link.h-cite > a") + val title = titleElement.text() + val url = titleElement.attr("href") + val tags = elem.select("span.tags > a").map(Element::text) + val (commentCount, commentsUrl) = getCommentsData(elem.select("span.comments_label")) + val submitter = + getSubmitter(elem.select("div.byline").first() ?: error("No byline element found")) + return LobstersPost( + shortId = shortId, + title = title, + url = url, + commentCount = commentCount, + commentsUrl = commentsUrl, + tags = tags, + submitter = submitter, + // The value of these fields is irrelevant for our use case + createdAt = "", + description = "", + ) + } + + private fun getCommentsData(elem: Elements): Pair { + val linkElement = elem.select("a") + val countString = linkElement.text().trimStart().substringBefore(" ") + val commentsUrl = linkElement.attr("href") + return (countString.toIntOrNull() ?: 0) to commentsUrl + } + + /** + * Make a bare-bones [User] object given a byline [elem]. We only need this to be usable for + * displaying in a list. + */ + private fun getSubmitter(elem: Element): User { + val userElement = elem.select("a.u-author") + val avatarElement = elem.select("img.avatar") + val username = userElement.text() + val avatarUrl = avatarElement.attr("src") + return User( + username = username, + about = "", + invitedBy = null, + avatarUrl = avatarUrl, + createdAt = "", + ) + } + + class Factory @Inject constructor(private val converter: SearchConverter) : Converter.Factory() { + override fun responseBodyConverter( + type: Type, + annotations: Array, + retrofit: Retrofit + ): Converter> { + return converter + } + } +} diff --git a/api/src/test/kotlin/dev/msfjarvis/claw/api/ApiWrapper.kt b/api/src/test/kotlin/dev/msfjarvis/claw/api/ApiWrapper.kt index 642fbc75..0e2c833c 100644 --- a/api/src/test/kotlin/dev/msfjarvis/claw/api/ApiWrapper.kt +++ b/api/src/test/kotlin/dev/msfjarvis/claw/api/ApiWrapper.kt @@ -14,7 +14,7 @@ import dev.msfjarvis.claw.model.LobstersPost import dev.msfjarvis.claw.model.LobstersPostDetails import dev.msfjarvis.claw.model.Tags import dev.msfjarvis.claw.model.User -import dev.msfjarvis.claw.util.TestUtils.getJson +import dev.msfjarvis.claw.util.TestUtils.getResource import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonNamingStrategy @@ -25,13 +25,13 @@ class ApiWrapper(controller: EitherNetController) { ignoreUnknownKeys = true namingStrategy = JsonNamingStrategy.SnakeCase } - private val hottest: List = json.decodeFromString(getJson("hottest.json")) + private val hottest: List = json.decodeFromString(getResource("hottest.json")) private val postDetails: LobstersPostDetails = - json.decodeFromString(getJson("post_details_tdfoqh.json")) - private val user: User = json.decodeFromString(getJson("msfjarvis.json")) - private val metaPosts: List = json.decodeFromString(getJson("meta.json")) + json.decodeFromString(getResource("post_details_tdfoqh.json")) + private val user: User = json.decodeFromString(getResource("msfjarvis.json")) + private val metaPosts: List = json.decodeFromString(getResource("meta.json")) private val programmingRustPosts: List = - json.decodeFromString(getJson("programming_rust.json")) + json.decodeFromString(getResource("programming_rust.json")) private val getPostsBody = { args: Array -> val tags = args[0] as Tags if ("meta" in tags) { diff --git a/api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiTest.kt b/api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiTest.kt new file mode 100644 index 00000000..5a1777d9 --- /dev/null +++ b/api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiTest.kt @@ -0,0 +1,59 @@ +/* + * Copyright © 2023 Harsh Shandilya. + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. + */ +package dev.msfjarvis.claw.api + +import com.google.common.truth.Truth.assertThat +import com.slack.eithernet.ApiResult +import com.slack.eithernet.test.newEitherNetController +import dev.msfjarvis.claw.model.LobstersPost +import dev.msfjarvis.claw.model.User +import dev.msfjarvis.claw.util.TestUtils.assertIs +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test + +class SearchApiTest { + private val wrapper = SearchApiWrapper(newEitherNetController()) + private val api + get() = wrapper.api + + @Test + fun `search is able to parse HTML`() = runTest { + val posts = api.searchPosts("ChatGPT", 1) + assertIs>>(posts) + assertThat(posts.value).containsAtLeastElementsIn(results).inOrder() + assertThat(posts.value).hasSize(20) + } + + private companion object { + private val results = + listOf( + LobstersPost( + shortId = "gjlsdg", + title = "ChatGPT visits the Emacs doctor", + url = "https://xenodium.com/chatgpt-visits-the-emacs-doctor/", + createdAt = "", + commentCount = 3, + commentsUrl = "/s/gjlsdg/chatgpt_visits_emacs_doctor", + submitter = User("xenodium", "", null, "/avatars/xenodium-16.png", ""), + tags = listOf("ai", "emacs"), + description = "", + ), + LobstersPost( + shortId = "astcqf", + title = + "Implementing a question-answering system for PDF documents using ChatGPT and Redis", + url = "https://mstack.nl/blog/20230623-chatgpt-question-pdf-document/", + createdAt = "", + commentCount = 0, + commentsUrl = "/s/astcqf/implementing_question_answering_system", + submitter = User("asteroid", "", null, "/avatars/asteroid-16.png", ""), + tags = listOf("ai"), + description = "", + ), + ) + } +} diff --git a/api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiWrapper.kt b/api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiWrapper.kt new file mode 100644 index 00000000..1d544784 --- /dev/null +++ b/api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiWrapper.kt @@ -0,0 +1,30 @@ +/* + * Copyright © 2023 Harsh Shandilya. + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. + */ +package dev.msfjarvis.claw.api + +import com.slack.eithernet.ApiResult.Companion.success +import com.slack.eithernet.test.EitherNetController +import com.slack.eithernet.test.enqueue +import dev.msfjarvis.claw.api.converters.SearchConverter +import dev.msfjarvis.claw.util.TestUtils.getResource +import okhttp3.MediaType +import okhttp3.ResponseBody + +class SearchApiWrapper(controller: EitherNetController) { + val api = controller.api + + init { + controller.enqueue(LobstersSearchApi::searchPosts) { + success( + SearchConverter() + .convert( + ResponseBody.create(MediaType.get("text/html"), getResource("search_chatgpt_page.html")) + ) + ) + } + } +} diff --git a/api/src/test/kotlin/dev/msfjarvis/claw/util/TestUtils.kt b/api/src/test/kotlin/dev/msfjarvis/claw/util/TestUtils.kt index e81abdb1..f8fc6ff1 100644 --- a/api/src/test/kotlin/dev/msfjarvis/claw/util/TestUtils.kt +++ b/api/src/test/kotlin/dev/msfjarvis/claw/util/TestUtils.kt @@ -13,7 +13,7 @@ import kotlin.contracts.contract @OptIn(ExperimentalContracts::class) object TestUtils { - fun getJson(path: String): String { + fun getResource(path: String): String { // Load the JSON response val uri = javaClass.classLoader!!.getResource(path) val file = File(uri.path) diff --git a/api/src/test/resources/search_chatgpt_page.html b/api/src/test/resources/search_chatgpt_page.html new file mode 100644 index 00000000..97df903e --- /dev/null +++ b/api/src/test/resources/search_chatgpt_page.html @@ -0,0 +1,5 @@ +Search | Lobsters

Search

 
   

225 results for "ChatGPT"

  1. 18
    ChatGPT visits the Emacs doctoraiemacsxenodium.com
    xenodium avatarauthored byxenodium3 days ago|suggest|flag|hide|save|
    Archive.orgArchive.todayGhostarchive
    |3 comments
  2. 5
    LLM CLI tool now supports self-hosted language models via pluginsaipythonsimonwillison.net
    simonw avatarauthored bysimonw3 days ago|suggest|flag|hide|save|
    Archive.orgArchive.todayGhostarchive
    |6 comments
  3. 1
    Implementing a question-answering system for PDF documents using ChatGPT and Redisaimstack.nl
    asteroid avatarviaasteroid6 days ago|suggest|flag|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
  4. 4
    A comparison of multiple generative AI tools when asking for Ruby on Rails codeairubyallaboutcoding.ghinda.com
    ebababi avatarviaebababi7 days ago|suggest|flag|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
  5. 1
    Experiencing Decreased Performance with ChatGPT-4aicommunity.openai.com
    kellogh avatarviakellogh8 days ago|suggest|flag|hide|save|
    Archive.orgArchive.todayGhostarchive
    |6 comments
  6. 18
    The Subtle Magic of tsnetvideogonetworkingtailscale.dev
    cadey avatarauthored bycadey8 days ago|suggest|flag|hide|save|
    Archive.orgArchive.todayGhostarchive
    |1 comment
  7. 4
    Searching for meaningainb.karmi.cz
    dmathieu avatarviadmathieu8 days ago|suggest|flag|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
  8. 5
    The LLMentalist Effect: how chat-based LLMs replicate the mechanisms of a psychic’s conaiculturesoftwarecrisis.dev
    breadbox avatarviabreadbox8 days ago|suggest|flag|hide|save|
    Archive.orgArchive.todayGhostarchive
    |2 comments
  9. 21
    How do you use AI development tools?☶askaiprogramming
    adamshaylor avatarauthored byadamshaylor9 days ago|suggest|flag|hide|save|29 comments
  10. 2
    QLoRA: Efficient Finetuning of Quantized LLMsaigithub.com
    sanxiyn avatarviasanxiyn15 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
  11. -2
    TailOps - less cloudy DevOps in the age of ChatGPTaidevopsnetworkingphilosophyprivacyvlad.git.ht
    fullmeta avatarauthored byfullmeta16 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |1 comment
  12. 5
    WinGPT: AI Assistant for Windows 3.1aiwindowsdialup.net
    fernplus avatarviafernplus16 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
  13. 1
    Awesome-LLM Github- a curated repository of LLModels, papers and evolution☶aigithub.com
    Yonz avatarviaYonz17 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
  14. -1
    Just Calm Down About GPT-4 Alreadyaicogscispectrum.ieee.org
    colonelpanic avatarviacolonelpanic22 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |1 comment
  15. 59
    Mozilla noticed a Google javascript and Linux kernel bugprogrammingfosstodon.org
    Vaelatern avatarviaVaelatern22 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
  16. 2
    One node module to integrate any AI modelaijavascriptnodejsgithub.com
    jaguar avatarauthored byjaguar23 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
  17. 4
    Navigating Sharp Edges in OpenAI's Function Call Featureaipythoncodeconfessions.substack.com
    abhi9u avatarauthored byabhi9u24 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
  18. 5
    Bing Chat: Data Exfiltration Exploit Explainedaisecurityembracethered.com
    river avatarviariver24 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
  19. 24
    People Are Using AI to Automate Responses to Site That Pays Them to Train AIaifuturism.com
    sharun avatarviasharun24 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |7 comments
  20. 2
    symbex: search Python code then pipe it into an LLMaipythonsimonwillison.net
    carlmjohnson avatarviacarlmjohnson26 days ago|suggest|hide|save|
    Archive.orgArchive.todayGhostarchive
    |no comments
Page:12345
\ No newline at end of file