Improve Search Performance and Result Quality

cmd/database/table/cdsViewField.go

@@ -50,6 +50,25 @@ func GetCDSViewFields(CDSViewTechnicalName string) (*[]CDSViewField, error) {
 	return &fields, nil
 }
 
+//go:embed sql/query_cds_view_number_of_fields.sql
+var query_cds_view_number_of_fields string
+
+func GetCDSViewNumberOfFields(CDSViewTechnicalName string) (int, error) {
+	rows, err := database.DB.Query(query_cds_view_number_of_fields, CDSViewTechnicalName)
+	if err != nil {
+		return 0, err
+	}
+
+	var numberOfFields int = 0
+	for rows.Next() {
+		err := rows.Scan(&numberOfFields)
+		if err != nil {
+			return 0, err
+		}
+	}
+	return numberOfFields, nil
+}
+
 //go:embed sql/insert_or_replace_cds_view_field.sql
 var insert_or_replace_cds_view_field string
 

cmd/database/table/sql/query_all_keywords.sql

@@ -1,16 +1,16 @@
-select
+select distinct
     CDSViewTechnicalName as CDSViewTechnicalName,
     lower(FieldName) as keyword
 from
     CDSViewField
 UNION
-select
+select distinct
     CDSViewTechnicalName as CDSViewTechnicalName,
     lower(Description) as keyword
 from
     CDSViewField
 UNION
-select
+select distinct
     TechnicalName as CDSViewTechnicalName,
     lower(TechnicalName) as keyword
 from
@@ -22,7 +22,7 @@ select
 from
     CDSView
 UNION
-select
+select distinct
     TechnicalName as CDSViewTechnicalName,
     lower(DisplayName) as keyword
 from

cmd/database/table/sql/query_cds_view_number_of_fields.sql

@@ -0,0 +1,6 @@
+SELECT
+    count(*) as NumberOfFields
+FROM
+    CDSViewField
+WHERE
+    CDSViewTechnicalName = ?

cmd/handler/cds.go

@@ -0,0 +1,23 @@
+package handler
+
+import (
+	"api-cds-search/cmd/model"
+	"api-cds-search/cmd/ui"
+	"fmt"
+	"net/http"
+)
+
+func GetCDSField(w http.ResponseWriter, r *http.Request) {
+	CDSViewTechnicalName := r.URL.Query().Get("CDSViewTechnicalName")
+	fields, err := model.GetCDSViewModelFields(CDSViewTechnicalName)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	err = ui.Template.ExecuteTemplate(w, "fields", fields)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+}

cmd/model/cdsView.go

@@ -19,6 +19,7 @@ package model
 
 import (
 	"api-cds-search/cmd/database/table"
+	"encoding/base64"
 	"fmt"
 	"strings"
 
@@ -36,23 +37,13 @@ type CDSViewFieldModel struct {
 type CDSViewModel struct {
 	*table.CDSView
 	StateTitle           string
-	Fields     *[]CDSViewFieldModel
+	TechnicalNameEncoded string
+	NumberOfFields       int
 }
 
 var englishCases = cases.Title(language.English)
 
-func GetCDSViewModel(TechnicalName string) (*CDSViewModel, error) {
-	var model CDSViewModel
-
-	cdsView, err := table.GetCDSView(TechnicalName)
-	if err != nil {
-		return nil, err
-	}
-
-	model.CDSView = cdsView
-
-	model.StateTitle = englishCases.String(model.State)
-
+func GetCDSViewModelFields(TechnicalName string) (*[]CDSViewFieldModel, error) {
 	fields, err := table.GetCDSViewFields(TechnicalName)
 	if err != nil {
 		return nil, err
@@ -71,7 +62,26 @@ func GetCDSViewModel(TechnicalName string) (*CDSViewModel, error) {
 		fieldsModel = append(fieldsModel, fieldModel)
 	}
 
-	model.Fields = &fieldsModel
+	return &fieldsModel, nil
+}
+
+func GetCDSViewModel(TechnicalName string) (*CDSViewModel, error) {
+	var model CDSViewModel
+
+	cdsView, err := table.GetCDSView(TechnicalName)
+	if err != nil {
+		return nil, err
+	}
+
+	model.CDSView = cdsView
+
+	model.StateTitle = englishCases.String(model.State)
+	model.TechnicalNameEncoded = strings.Replace(base64.StdEncoding.EncodeToString([]byte(model.TechnicalName)), "=", "", -1)
+
+	model.NumberOfFields, err = table.GetCDSViewNumberOfFields(TechnicalName)
+	if err != nil {
+		return nil, err
+	}
 
 	return &model, nil
 }

cmd/model/results.go

@@ -21,17 +21,26 @@ type ResultsModel struct {
 	SearchTerm  string
 	CurrentPage int
 	MaxPage     int
-	Views       []CDSViewModel
+	Views       []ResultView
+}
+
+type ResultView struct {
+	CDSViewModel
+	Score float32
 }
 
 type ResultsModelBuffer struct {
-	Views []CDSViewModel
+	Views []ResultView
 }
 
 func NewResultsModel() *ResultsModel {
 	return &ResultsModel{}
 }
 
-func (r *ResultsModel) AppendResultsModelViews(cdsView *CDSViewModel) {
-	r.Views = append(r.Views, *cdsView)
+func (r *ResultsModel) AppendResultsModelViews(cdsView *CDSViewModel, score float32) {
+	resultView := ResultView{
+		CDSViewModel: *cdsView,
+		Score:        score,
+	}
+	r.Views = append(r.Views, resultView)
 }

cmd/router/router.go

@@ -18,6 +18,7 @@
 package router
 
 import (
+	"api-cds-search/cmd/handler"
 	"api-cds-search/cmd/search"
 	"api-cds-search/cmd/ui"
 	"fmt"
@@ -34,6 +35,9 @@ func Load() *chi.Mux {
 		r.Route("/search", func(r chi.Router) {
 			r.Get("/", search.HandleSearch)
 		})
+		r.Route("/cds", func(r chi.Router) {
+			r.Get("/field/", handler.GetCDSField)
+		})
 	})
 
 	r.Handle("/ui/css/*", http.StripPrefix("/ui/css", http.FileServer(http.Dir("./cmd/ui/css"))))

cmd/search/search.go

@@ -110,30 +110,40 @@ func BuildBuffer(searchTerm string) error {
 
 	splitSearchTerms := strings.Split(searchTerm, " ")
 
+	if len(splitSearchTerms) > 1 {
 		splitSearchTerms = append(splitSearchTerms, searchTerm)
-	splitSearchTerms = append(splitSearchTerms, strings.ReplaceAll(searchTerm, " ", searchTerm))
+		splitSearchTerms = append(splitSearchTerms, strings.ReplaceAll(searchTerm, " ", ""))
+	}
 
 	var fuzzyResults []fuzzyResult
 
 	for viewName, targets := range searchTargets {
 		totalHits := 0
+		perfectHits := 0
+
 		var totalDistance float32
 		for i, splitSearchTerm := range splitSearchTerms {
+			if splitSearchTerm == "" {
+				continue
+			}
 			ranks := fuzzy.RankFindFold(splitSearchTerm, targets)
 
 			hits := len(ranks)
-			if hits == 0 {
+			/*if hits == 0 {
 			if i < len(splitSearchTerm)-3 {
 				totalDistance += 500
 			}
 			continue
-			}
+			}*/
 
 			totalHits += hits
 
 			var averageDistance float32 = 0
 			for _, rank := range ranks {
 				averageDistance += float32(rank.Distance)
+				if i >= len(splitSearchTerms)-2 && rank.Distance == 0 {
+					perfectHits += 1
+				}
 			}
 
 			totalDistance += averageDistance / float32(hits)
@@ -144,7 +154,7 @@ func BuildBuffer(searchTerm string) error {
 				CDSViewTechnicalName: viewName,
 				hits:                 totalHits,
 				averageDistance:      totalDistance,
-				score:                totalDistance / float32(totalHits),
+				score:                (float32(len(targets)) / float32(totalHits)) / float32(perfectHits),
 			})
 		}
 	}
@@ -160,6 +170,11 @@ func BuildBuffer(searchTerm string) error {
 		} else if a.score < b.score {
 			return -1
 		}
+		if a.CDSViewTechnicalName > b.CDSViewTechnicalName {
+			return 1
+		} else if a.CDSViewTechnicalName < b.CDSViewTechnicalName {
+			return -1
+		}
 		return 0
 	})
 
@@ -192,14 +207,22 @@ func HandleSearch(w http.ResponseWriter, r *http.Request) {
 	defer buffer.mu.Unlock()
 
 	page := GetPage(r)
+	var maxPage int
+	if buffer.hits <= pageStep {
+		maxPage = 0
+	} else {
+		maxPage = (buffer.hits / pageStep)
+	}
 
-	maxPage := (buffer.hits / pageStep) - 1
 	if page+1 > maxPage {
 		page = maxPage
 	}
 
 	resultsModel := model.NewResultsModel()
 	for i := page * pageStep; i < page*pageStep+pageStep; i++ {
+		if i > len(buffer.results)-1 {
+			break
+		}
 		result := buffer.results[i]
 		cdsView, err := model.GetCDSViewModel(result.CDSViewTechnicalName)
 		if err != nil {
@@ -207,7 +230,7 @@ func HandleSearch(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 
-		resultsModel.AppendResultsModelViews(cdsView)
+		resultsModel.AppendResultsModelViews(cdsView, result.score)
 	}
 
 	resultsModel.SearchTerm = searchTerm

cmd/ui/css/styles.css

@@ -120,6 +120,10 @@ body {
     min-width: calc(14 * var(--base_length));
 }
 
+.min-w24 {
+    min-width: calc(24 * var(--base_length));
+}
+
 .max-w1 {
     max-width: calc(1 * var(--base_length));
 }

cmd/ui/html/search-result.html

@@ -1,14 +1,20 @@
 {{block "results" .}}
-<div>
+<div class="max-w24" style="width: 100%">
     {{template "paging" .}}
+    <div class="container-center">
         <div class="boxed-list max-w24">
             {{range .Views}} {{template "result" .}} {{end}}
         </div>
+    </div>
 </div>
 
 {{end}} {{block "result" .}}
-<label class="active expander" for="{{.TechnicalName}}">
-    <input class="expander-state" type="checkbox" id="{{.TechnicalName}}" />
+<label class="active expander" for="{{.TechnicalNameEncoded}}">
+    <input
+        class="expander-state"
+        type="checkbox"
+        id="{{.TechnicalNameEncoded}}"
+    />
     <div class="title sans-serif">
         {{.DisplayName}}
         <div class="monospaced">{{.TechnicalName}}</div>
@@ -21,7 +27,9 @@
 </label>
 <div class="expander-content">
     <div class="label heading">Field Properties</div>
-    <div class="container-center">{{template "fields" .Fields}}</div>
+    <div class="container-center">
+        {{template "fields-placeholder" dict "Result" .}}
+    </div>
 </div>
 {{end}} {{block "fields" .}}
 <table>
@@ -33,6 +41,23 @@
     </tr>
     {{range .}} {{template "field" .}} {{end}}
 </table>
+{{end}} {{block "fields-placeholder" .}}
+<table
+    id="{{.Result.TechnicalNameEncoded}}_FieldPlaceholder"
+    hx-get="/cds/field/"
+    hx-vals='{"CDSViewTechnicalName":"{{.Result.TechnicalName}}"}'
+    hx-trigger="click from:#{{.Result.TechnicalNameEncoded}}"
+    hx-swap="outerHTML"
+    hx-target="this"
+>
+    <tr>
+        <td>
+            <div
+                style="height: calc(22px * ({{.Result.NumberOfFields}} + 1))"
+            ></div>
+        </td>
+    </tr>
+</table>
 {{end}} {{block "field" .}}
 <tr>
     <td class="table-field-left monospaced">{{.FieldName}}</td>
@@ -69,24 +94,7 @@
     <form style="float: right">
         <input name="q" disabled hidden value="{{.SearchTerm}}" />
         <div class="linked-horizontal">
-            <!--button
-                type="submit"
-                class="raised round"
-                style="margin-top: 9px"
-                hx-get="/search"
-                hx-params="*"
-                hx-include="#search-bar"
-                hx-target="#search-results"
-                hx-swap="innerHTML"
-                {{if
-                eq
-                .CurrentPage
-                0}}
-                disabled
-                {{end}}
-            >
-                ←</button
-            --><input
+            <input
                 class="round max-w2"
                 type="number"
                 name="p"
@@ -98,28 +106,12 @@
                 hx-include="#search-bar"
                 hx-target="#search-results"
                 hx-swap="innerHTML"
+                hx-throttle="1s"
             /><input
                 class="round raised max-w2"
                 value="{{.MaxPage}}"
                 disabled
-            /><!--button
-                type="submit"
-                class="raised round"
-                style="margin-top: 9px"
-                hx-get="/search"
-                hx-params="*"
-                hx-include="#search-bar"
-                hx-target="#search-results"
-                hx-swap="innerHTML"
-                {{if
-                eq
-                .CurrentPage
-                .MaxPage}}
-                disabled
-                {{end}}
-            >
-                →
-            </button-->
+            />
         </div>
     </form>
 </div>

cmd/ui/ui.go

@@ -17,13 +17,33 @@
 
 package ui
 
-import "text/template"
+import (
+	"errors"
+	"text/template"
+)
 
 var Template *template.Template
 
 func Load() {
-	Template = template.Must(template.New("ui").ParseFiles(
+	Template = template.Must(template.New("ui").Funcs(template.FuncMap{
+		"dict": Dict,
+	}).ParseFiles(
 		"./cmd/ui/html/main.html",
 		"./cmd/ui/html/search-result.html",
 	))
 }
+
+func Dict(values ...any) (map[string]any, error) {
+	if len(values)%2 != 0 {
+		return nil, errors.New("invalid dict call")
+	}
+	dict := make(map[string]any, len(values)/2)
+	for i := 0; i < len(values); i += 2 {
+		key, ok := values[i].(string)
+		if !ok {
+			return nil, errors.New("dict keys must be strings")
+		}
+		dict[key] = values[i+1]
+	}
+	return dict, nil
+}