From 49b31c237e4a2cf5be59b02ce9e62b993a3890ec Mon Sep 17 00:00:00 2001 From: Alec Strong Date: Sat, 2 Jun 2018 07:25:10 -0500 Subject: [PATCH] Use FTS to do partial matching --- .../sdksearch/search/ui/ItemViewHolder.kt | 16 +++++++-- .../sdksearch/store/ItemStoreTest.kt | 25 +++++++++++++- .../sdksearch/store/SqlItemStore.kt | 20 +++++------ .../jakewharton/sdksearch/store/QueryTerms.kt | 26 ++++++++++++++ .../com/jakewharton/sdksearch/store/Item.sq | 34 +++++++------------ store/src/main/sqldelight/migrations/4.sqm | 9 +++++ 6 files changed, 94 insertions(+), 36 deletions(-) create mode 100644 store/src/main/kotlin/com/jakewharton/sdksearch/store/QueryTerms.kt create mode 100644 store/src/main/sqldelight/migrations/4.sqm diff --git a/search/ui-android/src/main/java/com/jakewharton/sdksearch/search/ui/ItemViewHolder.kt b/search/ui-android/src/main/java/com/jakewharton/sdksearch/search/ui/ItemViewHolder.kt index 19fe3785..dbedc230 100644 --- a/search/ui-android/src/main/java/com/jakewharton/sdksearch/search/ui/ItemViewHolder.kt +++ b/search/ui-android/src/main/java/com/jakewharton/sdksearch/search/ui/ItemViewHolder.kt @@ -18,6 +18,7 @@ import androidx.core.text.inSpans import androidx.core.text.set import androidx.core.text.toSpannable import com.jakewharton.sdksearch.store.Item +import com.jakewharton.sdksearch.store.asTerms import kotlin.LazyThreadSafetyMode.NONE internal class ItemViewHolder( @@ -105,8 +106,19 @@ internal class ItemViewHolder( } val className = item.className.toSpannable() - val start = item.className.indexOf(query, ignoreCase = true) - className[start, start + query.length] = StyleSpan(BOLD) + query.asTerms().fold(0) { lastIndex, term -> + var start = item.className.indexOf(term, ignoreCase = term[0].isLowerCase(), startIndex = lastIndex) + if (start == -1) { + // Find it always ignoring case - there are some situations that we don't account for like + // the 'x' in R.xml + start = item.className.indexOf(term, ignoreCase = true, startIndex = lastIndex) + } + if (start == -1) { + throw IllegalStateException("Couldnt find term $term for item $item starting at $lastIndex") + } + className[start, start + term.length] = StyleSpan(BOLD) + return@fold start + 1 + } var dotIndex = item.className.indexOf('.') while (dotIndex >= 0) { diff --git a/store/android/src/androidTest/java/com/jakewharton/sdksearch/store/ItemStoreTest.kt b/store/android/src/androidTest/java/com/jakewharton/sdksearch/store/ItemStoreTest.kt index 366b141e..4ce96170 100644 --- a/store/android/src/androidTest/java/com/jakewharton/sdksearch/store/ItemStoreTest.kt +++ b/store/android/src/androidTest/java/com/jakewharton/sdksearch/store/ItemStoreTest.kt @@ -49,7 +49,6 @@ class ItemStoreTest { )) val item2 = query.receive().single() - assertEquals(id, item2.id) assertEquals("com.example", item2.packageName) assertEquals("One", item2.className) assertEquals("two.html", item2.link) @@ -100,4 +99,28 @@ class ItemStoreTest { it.cancel() } } + + @Test fun partialMatching() = runBlocking { + itemStore.updateItems(listOf( + ItemUtil.createForInsert("com.example.ConstraintLayout", "constraint.html", null), + ItemUtil.createForInsert("com.example.ConcurrentLinkedDeque", "concurrent.html", null) + )) + + itemStore.queryItems("ConL").also { + val results = it.receive() + assertEquals("ConstraintLayout", results[0].className) + assertEquals("ConcurrentLinkedDeque", results[1].className) + it.cancel() + } + + itemStore.queryItems("ConLa").also { + assertEquals("ConstraintLayout", it.receive().single().className) + it.cancel() + } + + itemStore.queryItems("Conla").also { + assertEquals(0, it.receive().size) + it.cancel() + } + } } diff --git a/store/android/src/main/java/com/jakewharton/sdksearch/store/SqlItemStore.kt b/store/android/src/main/java/com/jakewharton/sdksearch/store/SqlItemStore.kt index 17c84420..e25415a4 100644 --- a/store/android/src/main/java/com/jakewharton/sdksearch/store/SqlItemStore.kt +++ b/store/android/src/main/java/com/jakewharton/sdksearch/store/SqlItemStore.kt @@ -6,6 +6,7 @@ import com.squareup.sqldelight.runtime.coroutines.mapToList import javax.inject.Inject import javax.inject.Singleton import kotlin.coroutines.experimental.CoroutineContext +import kotlinx.coroutines.experimental.channels.ReceiveChannel @Singleton internal class SqlItemStore @Inject constructor( @@ -15,21 +16,18 @@ internal class SqlItemStore @Inject constructor( override suspend fun updateItems(items: List) { db.transaction { for (item in items) { - val affected = db.updateItem(item.deprecated, item.link, item.packageName, item.className) - if (affected == 0L) { - db.insertItem(item.packageName, item.className, item.deprecated, item.link) - } + db.insertItem(item.packageName, item.className, item.deprecated, item.link) + db.insertItemIndex(item.className.asTerms().joinToString(" ")) } } } - override fun queryItems(term: String) = - db.queryTerm(term.escapeLike('\\')).asChannel(context).mapToList() - - private fun String.escapeLike(escapeChar: Char) = - this.replace("$escapeChar", "$escapeChar$escapeChar") - .replace("%", "$escapeChar%") - .replace("_", "${escapeChar}_") + override fun queryItems(term: String): ReceiveChannel> { + val terms = term.asTerms().joinToString(" ") { "$it*" } + return db.queryTerm("\"$terms\"") + .asChannel(context) + .mapToList() + } override fun count() = db.count().asChannel(context).mapToOne() } diff --git a/store/src/main/kotlin/com/jakewharton/sdksearch/store/QueryTerms.kt b/store/src/main/kotlin/com/jakewharton/sdksearch/store/QueryTerms.kt new file mode 100644 index 00000000..6ea1f014 --- /dev/null +++ b/store/src/main/kotlin/com/jakewharton/sdksearch/store/QueryTerms.kt @@ -0,0 +1,26 @@ +package com.jakewharton.sdksearch.store + +fun String.asTerms(): List { + var splitNext = false + return this.fold(emptyList()) { segments, character -> + // Fts cant handle these. Encode them as their unicode + when (character) { + '%' -> return@fold segments + "U0025" + '_' -> return@fold segments + "U005F" + '\\' -> return@fold segments + "U005C" + } + + if (character !in 'A'..'Z' && character !in 'a'..'z') { + splitNext = true + return@fold segments + } + + + val lastSegment = segments.lastOrNull() ?: return@fold listOf(character.toString()) + if (character.toLowerCase() == character && !splitNext) { + return@fold segments.dropLast(1) + (lastSegment + character) + } + splitNext = false + return@fold segments + character.toString() + } +} diff --git a/store/src/main/sqldelight/com/jakewharton/sdksearch/store/Item.sq b/store/src/main/sqldelight/com/jakewharton/sdksearch/store/Item.sq index c56b24e5..6cc6a828 100644 --- a/store/src/main/sqldelight/com/jakewharton/sdksearch/store/Item.sq +++ b/store/src/main/sqldelight/com/jakewharton/sdksearch/store/Item.sq @@ -10,31 +10,21 @@ CREATE TABLE item( CREATE VIRTUAL TABLE item_index USING fts4(content TEXT); -CREATE TRIGGER populate_index -AFTER INSERT ON item +CREATE TRIGGER delete_index +AFTER DELETE ON item BEGIN - INSERT INTO item_index (docid, content) - VALUES (new.id, new.className); -END; - -CREATE TRIGGER update_index -AFTER UPDATE ON item -BEGIN - UPDATE item_index - SET content = new.className - WHERE docid = new.id; -END; + DELETE FROM item_index + WHERE docid = old.id; +END +; -insertItem: -INSERT OR FAIL INTO item(packageName, className, deprecated, link) VALUES (?, ?, ?, ?) +insertItemIndex: +INSERT INTO item_index (docid, content) +VALUES (last_insert_rowid(), ('' || ?)) -- https://github.com/square/sqldelight/pull/863 ; -updateItem: -UPDATE item -SET deprecated = ?3, - link = ?4 -WHERE packageName = ?1 - AND className = ?2 +insertItem: +INSERT OR REPLACE INTO item(packageName, className, deprecated, link) VALUES (?, ?, ?, ?) ; count: @@ -46,7 +36,7 @@ queryTerm: SELECT item.* FROM item_index JOIN item ON (docid = item.id) -WHERE content LIKE '%' || ?1 || '%' ESCAPE '\' +WHERE content MATCH ('' || ?1) -- https://github.com/square/sqldelight/pull/863 ORDER BY -- deprecated classes are always last deprecated ASC, diff --git a/store/src/main/sqldelight/migrations/4.sqm b/store/src/main/sqldelight/migrations/4.sqm new file mode 100644 index 00000000..66063f58 --- /dev/null +++ b/store/src/main/sqldelight/migrations/4.sqm @@ -0,0 +1,9 @@ +DROP TRIGGER populate_index; +DROP TRIGGER update_index; + +CREATE TRIGGER delete_index +AFTER DELETE ON item +BEGIN + DELETE FROM item_index + WHERE docid = old.id; +END;