refactor(android): move link metadata extraction to Unfurl

Crux is doing major changes for no good reason

android/build.gradle.kts

@@ -71,7 +71,6 @@ dependencies {
   implementation(libs.androidx.work.runtime)
   implementation(libs.coil)
   implementation(libs.copydown)
-  implementation(libs.crux)
   implementation(libs.dagger)
   implementation(libs.jsoup)
   implementation(libs.kotlinx.collections.immutable)
@@ -79,6 +78,7 @@ dependencies {
   implementation(libs.napier)
   implementation(libs.soloader)
   implementation(libs.sqldelight.extensions.coroutines)
+  implementation(libs.unfurl)
   implementation(projects.api)
   implementation(projects.common)
   implementation(projects.core)

android/src/main/kotlin/dev/msfjarvis/claw/android/injection/MetadataExtractorModule.kt

@@ -1,25 +1,44 @@
 /*
- * Copyright © 2021-2022 Harsh Shandilya.
+ * Copyright © 2021-2023 Harsh Shandilya.
  * Use of this source code is governed by an MIT-style
  * license that can be found in the LICENSE file or at
  * https://opensource.org/licenses/MIT.
  */
 package dev.msfjarvis.claw.android.injection
 
-import com.chimbori.crux.Crux
 import com.deliveryhero.whetstone.app.ApplicationScope
 import com.squareup.anvil.annotations.ContributesTo
 import dagger.Module
 import dagger.Provides
+import io.github.aakira.napier.Napier
+import me.saket.unfurl.UnfurlLogger
+import me.saket.unfurl.Unfurler
 import okhttp3.OkHttpClient
 
 @Module
 @ContributesTo(ApplicationScope::class)
 object MetadataExtractorModule {
   @Provides
-  fun provideCrux(
+  fun provideUnfurlLogger(): UnfurlLogger {
+    return object : UnfurlLogger {
+      override fun log(message: String) {
+        Napier.d { message }
+      }
+
+      override fun log(e: Throwable, message: String) {
+        Napier.e(e) { message }
+      }
+    }
+  }
+
+  @Provides
+  fun provideUnfurler(
     okHttpClient: OkHttpClient,
-  ): Crux {
+    logger: UnfurlLogger,
-    return Crux(okHttpClient = okHttpClient)
+  ): Unfurler {
+    return Unfurler(
+      httpClient = okHttpClient,
+      logger = logger,
+    )
   }
 }

android/src/main/kotlin/dev/msfjarvis/claw/android/viewmodel/LinkMetadataRepository.kt

@@ -6,44 +6,21 @@
  */
 package dev.msfjarvis.claw.android.viewmodel
 
-import com.chimbori.crux.Crux
+import dev.msfjarvis.claw.core.injection.IODispatcher
-import com.chimbori.crux.api.Fields
 import dev.msfjarvis.claw.model.LinkMetadata
 import javax.inject.Inject
-import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+import kotlinx.coroutines.CoroutineDispatcher
-import okhttp3.OkHttpClient
+import kotlinx.coroutines.withContext
-import okhttp3.Request
+import me.saket.unfurl.Unfurler
-import org.jsoup.Jsoup
 
 class LinkMetadataRepository
 @Inject
 constructor(
-  private val crux: Crux,
+  private val unfurler: Unfurler,
-  private val okHttpClient: OkHttpClient,
+  @IODispatcher private val dispatcher: CoroutineDispatcher,
 ) {
   suspend fun getLinkMetadata(url: String): LinkMetadata {
-    return run {
+    val result = withContext(dispatcher) { unfurler.unfurl(url) }
-      val parsedUrl = url.toHttpUrlOrNull() ?: return@run null
+    return LinkMetadata(url, result?.favicon?.toString())
-      if (!parsedUrl.isHttps) return@run null
-      val request = Request.Builder().url(parsedUrl).build()
-      val htmlContent =
-        okHttpClient.newCall(request).execute().use { response ->
-          val body = response.body ?: return@run null
-          body.string()
-        }
-      val extractedMetadata = crux.extractFrom(parsedUrl, Jsoup.parse(htmlContent, url))
-      val faviconUrl = extractedMetadata[Fields.FAVICON_URL].toString()
-      val readingTime = extractedMetadata[Fields.DURATION_MS].toString()
-      LinkMetadata(
-        url = url,
-        faviconUrl = faviconUrl,
-        readingTime = readingTime,
-      )
-    }
-      ?: LinkMetadata(
-        url = url,
-        faviconUrl = null,
-        readingTime = null,
-      )
   }
 }

common/src/main/kotlin/dev/msfjarvis/claw/common/comments/CommentEntry.kt

@@ -59,7 +59,7 @@ internal fun CommentsHeader(
   val uriHandler = LocalUriHandler.current
   val linkMetadata by
     produceState(
-      initialValue = LinkMetadata(postDetails.url, null, null),
+      initialValue = LinkMetadata(postDetails.url, null),
     ) {
       runCatching { postActions.getLinkMetadata(postDetails.url) }
         .onSuccess { metadata -> value = metadata }

common/src/main/kotlin/dev/msfjarvis/claw/common/posts/LobstersCard.kt

@@ -289,7 +289,7 @@ fun LobstersCardPreview() {
           }
 
           override suspend fun getLinkMetadata(url: String): LinkMetadata {
-            return LinkMetadata("", "", "")
+            return LinkMetadata("", "")
           }
         },
     )

gradle/libs.versions.toml

@@ -61,7 +61,6 @@ compose-richtext-markdown = { module = "com.halilibo.compose-richtext:richtext-c
 compose-richtext-material3 = { module = "com.halilibo.compose-richtext:richtext-ui-material3", version.ref = "richtext" }
 compose-richtext-ui = { module = "com.halilibo.compose-richtext:richtext-ui", version.ref = "richtext" }
 copydown = "io.github.furstenheim:copy_down:1.1"
-crux = "com.chimbori.crux:crux:5.0"
 dagger = { module = "com.google.dagger:dagger", version.ref = "dagger" }
 dagger-compiler = { module = "com.google.dagger:dagger-compiler", version.ref = "dagger" }
 eithernet = "com.slack.eithernet:eithernet:1.4.1"
@@ -93,6 +92,7 @@ sqldelight-primitiveAdapters = { module = "app.cash.sqldelight:primitive-adapter
 sqlite-android = "com.github.requery:sqlite-android:3.42.0"
 testparameterinjector = "com.google.testparameterinjector:test-parameter-injector:1.12"
 truth = "com.google.truth:truth:1.1.5"
+unfurl = "me.saket.unfurl:unfurl:1.7.0"
 whetstone = { module = "com.deliveryhero.whetstone:whetstone", version.ref = "whetstone" }
 
 [plugins]

model/src/main/kotlin/dev/msfjarvis/claw/model/LinkMetadata.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2022 Harsh Shandilya.
+ * Copyright © 2022-2023 Harsh Shandilya.
  * Use of this source code is governed by an MIT-style
  * license that can be found in the LICENSE file or at
  * https://opensource.org/licenses/MIT.
@@ -8,8 +8,10 @@
 
 package dev.msfjarvis.claw.model
 
-data class LinkMetadata(
+import dev.drewhamilton.poko.Poko
+
+@Poko
+class LinkMetadata(
   val url: String,
   val faviconUrl: String?,
-  val readingTime: String?,
 )