feat(api): add a Retrofit service for search

api/build.gradle.kts

@@ -24,6 +24,7 @@ dependencies {
 
   implementation(libs.dagger)
   implementation(libs.javax.inject)
+  implementation(libs.jsoup)
 
   testImplementation(testFixtures(libs.eithernet))
   testImplementation(libs.kotlinx.coroutines.test)

api/src/main/kotlin/dev/msfjarvis/claw/api/LobstersSearchApi.kt

@@ -0,0 +1,20 @@
+/*
+ * Copyright © 2023 Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.api
+
+import com.slack.eithernet.ApiResult
+import dev.msfjarvis.claw.model.LobstersPost
+import retrofit2.http.GET
+import retrofit2.http.Query
+
+interface LobstersSearchApi {
+  @GET("/search?q={query}&what=stories&order=newest&page={page}")
+  suspend fun searchPosts(
+    @Query("query") searchQuery: String,
+    @Query("page") page: Int,
+  ): ApiResult<List<LobstersPost>, Unit>
+}

api/src/main/kotlin/dev/msfjarvis/claw/api/converters/SearchConverter.kt

@@ -0,0 +1,86 @@
+/*
+ * Copyright © 2023 Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.api.converters
+
+import dev.msfjarvis.claw.api.LobstersApi
+import dev.msfjarvis.claw.model.LobstersPost
+import dev.msfjarvis.claw.model.User
+import java.lang.reflect.Type
+import javax.inject.Inject
+import okhttp3.ResponseBody
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Element
+import org.jsoup.select.Elements
+import retrofit2.Converter
+import retrofit2.Retrofit
+
+class SearchConverter @Inject constructor() : Converter<ResponseBody, List<LobstersPost>> {
+  override fun convert(value: ResponseBody): List<LobstersPost> {
+    val elements =
+      Jsoup.parse(value.string(), LobstersApi.BASE_URL).select("div.story_liner.h-entry")
+    return elements.map(::parsePost)
+  }
+
+  private fun parsePost(elem: Element): LobstersPost {
+    val parent = elem.parent() ?: error("$elem must have a parent")
+    val shortId = parent.attr("data-shortid")
+    val titleElement = elem.select("span.link.h-cite > a")
+    val title = titleElement.text()
+    val url = titleElement.attr("href")
+    val tags = elem.select("span.tags > a").map(Element::text)
+    val (commentCount, commentsUrl) = getCommentsData(elem.select("span.comments_label"))
+    val submitter =
+      getSubmitter(elem.select("div.byline").first() ?: error("No byline element found"))
+    return LobstersPost(
+      shortId = shortId,
+      title = title,
+      url = url,
+      commentCount = commentCount,
+      commentsUrl = commentsUrl,
+      tags = tags,
+      submitter = submitter,
+      // The value of these fields is irrelevant for our use case
+      createdAt = "",
+      description = "",
+    )
+  }
+
+  private fun getCommentsData(elem: Elements): Pair<Int, String> {
+    val linkElement = elem.select("a")
+    val countString = linkElement.text().trimStart().substringBefore(" ")
+    val commentsUrl = linkElement.attr("href")
+    return (countString.toIntOrNull() ?: 0) to commentsUrl
+  }
+
+  /**
+   * Make a bare-bones [User] object given a byline [elem]. We only need this to be usable for
+   * displaying in a list.
+   */
+  private fun getSubmitter(elem: Element): User {
+    val userElement = elem.select("a.u-author")
+    val avatarElement = elem.select("img.avatar")
+    val username = userElement.text()
+    val avatarUrl = avatarElement.attr("src")
+    return User(
+      username = username,
+      about = "",
+      invitedBy = null,
+      avatarUrl = avatarUrl,
+      createdAt = "",
+    )
+  }
+
+  class Factory @Inject constructor(private val converter: SearchConverter) : Converter.Factory() {
+    override fun responseBodyConverter(
+      type: Type,
+      annotations: Array<out Annotation>,
+      retrofit: Retrofit
+    ): Converter<ResponseBody, List<LobstersPost>> {
+      return converter
+    }
+  }
+}

api/src/test/kotlin/dev/msfjarvis/claw/api/ApiWrapper.kt

@@ -14,7 +14,7 @@ import dev.msfjarvis.claw.model.LobstersPost
 import dev.msfjarvis.claw.model.LobstersPostDetails
 import dev.msfjarvis.claw.model.Tags
 import dev.msfjarvis.claw.model.User
-import dev.msfjarvis.claw.util.TestUtils.getJson
+import dev.msfjarvis.claw.util.TestUtils.getResource
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.JsonNamingStrategy
@@ -25,13 +25,13 @@ class ApiWrapper(controller: EitherNetController<LobstersApi>) {
     ignoreUnknownKeys = true
     namingStrategy = JsonNamingStrategy.SnakeCase
   }
-  private val hottest: List<LobstersPost> = json.decodeFromString(getJson("hottest.json"))
+  private val hottest: List<LobstersPost> = json.decodeFromString(getResource("hottest.json"))
   private val postDetails: LobstersPostDetails =
-    json.decodeFromString(getJson("post_details_tdfoqh.json"))
-  private val user: User = json.decodeFromString(getJson("msfjarvis.json"))
-  private val metaPosts: List<LobstersPost> = json.decodeFromString(getJson("meta.json"))
+    json.decodeFromString(getResource("post_details_tdfoqh.json"))
+  private val user: User = json.decodeFromString(getResource("msfjarvis.json"))
+  private val metaPosts: List<LobstersPost> = json.decodeFromString(getResource("meta.json"))
   private val programmingRustPosts: List<LobstersPost> =
-    json.decodeFromString(getJson("programming_rust.json"))
+    json.decodeFromString(getResource("programming_rust.json"))
   private val getPostsBody = { args: Array<Any> ->
     val tags = args[0] as Tags
     if ("meta" in tags) {

api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiTest.kt

@@ -0,0 +1,59 @@
+/*
+ * Copyright © 2023 Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.api
+
+import com.google.common.truth.Truth.assertThat
+import com.slack.eithernet.ApiResult
+import com.slack.eithernet.test.newEitherNetController
+import dev.msfjarvis.claw.model.LobstersPost
+import dev.msfjarvis.claw.model.User
+import dev.msfjarvis.claw.util.TestUtils.assertIs
+import kotlinx.coroutines.test.runTest
+import org.junit.jupiter.api.Test
+
+class SearchApiTest {
+  private val wrapper = SearchApiWrapper(newEitherNetController())
+  private val api
+    get() = wrapper.api
+
+  @Test
+  fun `search is able to parse HTML`() = runTest {
+    val posts = api.searchPosts("ChatGPT", 1)
+    assertIs<ApiResult.Success<List<LobstersPost>>>(posts)
+    assertThat(posts.value).containsAtLeastElementsIn(results).inOrder()
+    assertThat(posts.value).hasSize(20)
+  }
+
+  private companion object {
+    private val results =
+      listOf(
+        LobstersPost(
+          shortId = "gjlsdg",
+          title = "ChatGPT visits the Emacs doctor",
+          url = "https://xenodium.com/chatgpt-visits-the-emacs-doctor/",
+          createdAt = "",
+          commentCount = 3,
+          commentsUrl = "/s/gjlsdg/chatgpt_visits_emacs_doctor",
+          submitter = User("xenodium", "", null, "/avatars/xenodium-16.png", ""),
+          tags = listOf("ai", "emacs"),
+          description = "",
+        ),
+        LobstersPost(
+          shortId = "astcqf",
+          title =
+            "Implementing a question-answering system for PDF documents using ChatGPT and Redis",
+          url = "https://mstack.nl/blog/20230623-chatgpt-question-pdf-document/",
+          createdAt = "",
+          commentCount = 0,
+          commentsUrl = "/s/astcqf/implementing_question_answering_system",
+          submitter = User("asteroid", "", null, "/avatars/asteroid-16.png", ""),
+          tags = listOf("ai"),
+          description = "",
+        ),
+      )
+  }
+}

api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiWrapper.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright © 2023 Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.api
+
+import com.slack.eithernet.ApiResult.Companion.success
+import com.slack.eithernet.test.EitherNetController
+import com.slack.eithernet.test.enqueue
+import dev.msfjarvis.claw.api.converters.SearchConverter
+import dev.msfjarvis.claw.util.TestUtils.getResource
+import okhttp3.MediaType
+import okhttp3.ResponseBody
+
+class SearchApiWrapper(controller: EitherNetController<LobstersSearchApi>) {
+  val api = controller.api
+
+  init {
+    controller.enqueue(LobstersSearchApi::searchPosts) {
+      success(
+        SearchConverter()
+          .convert(
+            ResponseBody.create(MediaType.get("text/html"), getResource("search_chatgpt_page.html"))
+          )
+      )
+    }
+  }
+}

api/src/test/kotlin/dev/msfjarvis/claw/util/TestUtils.kt

@@ -13,7 +13,7 @@ import kotlin.contracts.contract
 
 @OptIn(ExperimentalContracts::class)
 object TestUtils {
-  fun getJson(path: String): String {
+  fun getResource(path: String): String {
     // Load the JSON response
     val uri = javaClass.classLoader!!.getResource(path)
     val file = File(uri.path)