feat(update_database): improvements and preparations before further development

* doc(init_database): added todos with plans

* feat(init_database): pulled methods to utils and added stubs for future

* feat(update_database): added path_id to inheritance table

* feat(drop_by_path): for future implementation of update_database

* fix: lint and style

* chore(halt): cancelling for now further development

* fix(ci): double test execution

---------

Co-authored-by: Sasha Gurevich <sagurevich@microsoft.com>

Author: leblocks

.github/workflows/lint.yml

@@ -1,6 +1,6 @@
 name: Linting and style checking
 
-on: [push, pull_request]
+on: [push]
 
 jobs:
   luacheck:

lua/hopcsharp/database/buffer.lua

@@ -1,7 +1,9 @@
 local BufferedWriter = {}
 BufferedWriter.__index = BufferedWriter
 
--- Constructor
+---@class BufferedWriter
+---TODO finish
+
 ---@param db sqlite_db db object
 ---@param buffer_size number size of the buffer
 function BufferedWriter:new(db, buffer_size)

lua/hopcsharp/database/init.lua

@@ -23,6 +23,7 @@ M.__init_db = function()
             column = 'integer',
         },
         inheritance = {
+            path_id = { type = 'integer', reference = 'files.id' },
             name = 'text',
             base = 'text',
         },
@@ -51,4 +52,26 @@ M.__drop_db = function()
     db:eval('vacuum')
 end
 
+M.__drop_by_path = function(paths)
+    if paths == nil or #paths == 0 then
+        return
+    end
+
+    local db = M.__get_db()
+    local files = db:select('files', { where = { path = paths } })
+
+    if #files == 0 then
+        return
+    end
+
+    local ids = {}
+    for _, file in ipairs(files) do
+        table.insert(ids, file.id)
+    end
+
+    db:delete('files', { where = { id = ids } })
+    db:delete('inheritance', { where = { path_id = ids } })
+    db:delete('definitions', { where = { path_id = ids } })
+end
+
 return M

lua/hopcsharp/init.lua

@@ -12,36 +12,19 @@ local BufferedWriter = require('hopcsharp.database.buffer')
 
 local M = {}
 
-local PROCESSING_ERROR_MESSAGE = 'init_database is running, try again later. '
-    .. 'If init_database failed - restart or manually set vim.g.hopcsharp_processing to false'
-
 vim.g.hopcsharp_processing = false
 
-local function log(message, prefix)
-    prefix = prefix or 'hopcsharp: '
-    print(prefix .. message)
-end
-
-local function scheduled_iteration(i, iterable, callback)
-    if i > #iterable then
-        return
-    end
-
-    callback(i, iterable)
-
-    vim.schedule(function()
-        scheduled_iteration(i + 1, iterable, callback)
-    end)
-end
-
 M.__init_database = function()
     -- drop existing schema
     database.__drop_db()
     local writer = BufferedWriter:new(database.__get_db(), 1000)
 
+    local files = parse.__get_source_files()
+
     local counter = 0
-    scheduled_iteration(1, parse.__get_source_files(), function(i, items)
-        parse.__parse_tree(items[i], function(tree, file_path_id, file_content, wr)
+
+    utils.__scheduled_iteration(files, function(i, item, items)
+        parse.__parse_tree(item, function(tree, file_path_id, file_content, wr)
             local root = tree:root()
             definition.__parse_definitions(root, file_path_id, file_content, wr)
             inheritance.__parse_inheritance(root, file_path_id, file_content, wr)
@@ -50,7 +33,7 @@ M.__init_database = function()
         counter = counter + 1
 
         if counter % 100 == 0 then
-            log(string.format('processed %s/%s of files', counter, #items), '')
+            utils.__log(string.format('processed %s/%s of files', counter, #items), '')
         end
 
         if counter == #items then
@@ -60,93 +43,82 @@ M.__init_database = function()
 end
 
 M.init_database = function()
-    if vim.g.hopcsharp_processing then
-        vim.notify(PROCESSING_ERROR_MESSAGE)
-        return
-    end
-
-    vim.g.hopcsharp_processing = true
-
-    local command = {
-        'nvim',
-        '--headless',
-        '-c',
-        'lua require("hopcsharp").__init_database()',
-        '-c',
-        'qa',
-    }
-
-    local line_buffer = {}
-    -- using table here, for quirks of different OS's
-    -- to add different line separators
-    local line_separators = { '\r\r', '\r' }
-
-    local function flush_line_buffer()
-        if #line_buffer > 0 then
-            log(table.concat(line_buffer, ''))
-            line_buffer = {}
+    utils.__block_on_processing(function()
+        vim.g.hopcsharp_processing = true
+
+        local command = {
+            'nvim',
+            '--headless',
+            '-c',
+            'lua require("hopcsharp").__init_database()',
+            '-c',
+            'qa',
+        }
+
+        local line_buffer = {}
+        -- using table here, for quirks of different OS's
+        -- to add different line separators
+        local line_separators = { '\r\r', '\r' }
+
+        local function flush_line_buffer()
+            if #line_buffer > 0 then
+                utils.__log(table.concat(line_buffer, ''))
+                line_buffer = {}
+            end
         end
-    end
-
-    local start = os.time()
-    local on_stdout = function(_, data)
-        for _, entry in ipairs(data) do
-            if utils.__contains(line_separators, entry) then
-                flush_line_buffer()
-            else
-                table.insert(line_buffer, entry)
+
+        local start = os.time()
+        local on_stdout = function(_, data)
+            for _, entry in ipairs(data) do
+                if utils.__contains(line_separators, entry) then
+                    flush_line_buffer()
+                else
+                    table.insert(line_buffer, entry)
+                end
             end
         end
-    end
-
-    local on_exit = function(_)
-        flush_line_buffer()
-        vim.g.hopcsharp_processing = false
-        local elapsed = os.difftime(os.time(), start)
-        log('finished processing files ' .. elapsed .. 's elapsed')
-    end
-
-    -- spawn actual parsing in a separate instance of neovim
-    vim.fn.jobstart(command, {
-        cwd = vim.fn.getcwd(),
-        on_stdout = on_stdout,
-        on_exit = on_exit,
-        pty = true,
-        stdin = nil,
-    })
+
+        local on_exit = function(_)
+            flush_line_buffer()
+            vim.g.hopcsharp_processing = false
+            local elapsed = os.difftime(os.time(), start)
+            utils.__log('finished processing files ' .. elapsed .. 's elapsed')
+        end
+
+        -- spawn actual parsing in a separate instance of neovim
+        vim.fn.jobstart(command, {
+            cwd = vim.fn.getcwd(),
+            on_stdout = on_stdout,
+            on_exit = on_exit,
+            pty = true,
+            stdin = nil,
+        })
+    end)
 end
 
 M.hop_to_definition = function(config)
-    if vim.g.hopcsharp_processing then
-        vim.notify(PROCESSING_ERROR_MESSAGE)
-        return
-    end
-    hop.__hop_to_definition(config)
+    utils.__block_on_processing(function()
+        hop.__hop_to_definition(config)
+    end)
 end
 
 M.hop_to_implementation = function(config)
-    if vim.g.hopcsharp_processing then
-        vim.notify(PROCESSING_ERROR_MESSAGE)
-        return
-    end
-    hop.__hop_to_implementation(config)
+    utils.__block_on_processing(function()
+        hop.__hop_to_implementation(config)
+    end)
 end
 
 M.get_type_hierarchy = function()
-    if vim.g.hopcsharp_processing then
-        vim.notify(PROCESSING_ERROR_MESSAGE)
-        return
-    end
-    hierarchy.__get_type_hierarchy()
+    utils.__block_on_processing(function()
+        hierarchy.__get_type_hierarchy()
+    end)
 end
 
 ---@return sqlite_db
 M.get_db = function()
-    if vim.g.hopcsharp_processing then
-        vim.notify(PROCESSING_ERROR_MESSAGE)
-        return
-    end
-    return database.__get_db()
+    return utils.__block_on_processing(function()
+        return database.__get_db()
+    end)
 end
 
 return M

lua/hopcsharp/parse/inheritance.lua

@@ -4,11 +4,15 @@ local query = require('hopcsharp.parse.query')
 local M = {}
 
 ---@param tree TSNode under which the search will occur
+---@param path_id number file path id
 ---@param file_content string file content
 ---@param writer BufferedWriter buffered database writer
-M.__parse_inheritance = function(tree, _, file_content, writer)
+M.__parse_inheritance = function(tree, path_id, file_content, writer)
     for _, match, _ in query.base_identifier:iter_matches(tree, file_content, 0, -1) do
-        local entry = {}
+        local entry = {
+            path_id = path_id,
+        }
+
         for id, nodes in pairs(match) do
             local name = query.base_identifier.captures[id]
             for _, node in ipairs(nodes) do

lua/hopcsharp/utils.lua

@@ -31,4 +31,44 @@ M.__trim_spaces = function(word)
     return string.gsub(word, '%s+', '')
 end
 
+M.__block_on_processing = function(callback)
+    local message = 'init_database is running, try again later. '
+        .. 'If init_database failed - restart or manually set vim.g.hopcsharp_processing to false'
+
+    if vim.g.hopcsharp_processing then
+        vim.notify(message)
+        return
+    end
+
+    return callback()
+end
+
+M.__log = function(message, prefix)
+    prefix = prefix or 'hopcsharp: '
+    print(prefix .. message)
+end
+
+---@param entries table to iterate on
+---@param callback function will be called on each entry in entries via vim.schedule
+M.__scheduled_iteration = function(entries, callback)
+    if not entries then
+        return
+    end
+
+    local function iterate(i)
+        if i > #entries then
+            -- finished iteration
+            return
+        end
+
+        callback(i, entries[i], entries)
+
+        vim.schedule(function()
+            iterate(i + 1)
+        end)
+    end
+
+    iterate(1)
+end
+
 return M

test/database/database_spec.lua

@@ -1,4 +1,5 @@
 local database = require('hopcsharp.database')
+local utils = require('test.utils')
 
 describe('database', function()
     it('__init_db returns db object', function()
@@ -9,4 +10,46 @@ describe('database', function()
     it('__drop_db can be called', function()
         database.__drop_db()
     end)
+
+    it('__drop_by_path - empty path', function()
+        utils.init_test_database()
+        database.__drop_by_path({})
+    end)
+
+    it('__drop_by_path - nil path', function()
+        utils.init_test_database()
+        database.__drop_by_path(nil)
+    end)
+
+    it('__drop_by_path - non existing paths', function()
+        utils.init_test_database()
+        database.__drop_by_path({
+            vim.fs.normalize('meow//woof'),
+            vim.fs.normalize('woofs//meow'),
+        })
+    end)
+
+    it('__drop_by_path - happy path', function()
+        utils.init_test_database()
+
+        local paths = {
+            vim.fs.normalize('test\\sources\\get_type_hierarchy.cs'),
+            vim.fs.normalize('test\\sources\\hop_to_definition.cs'),
+        }
+
+        local db = database.__get_db()
+        local files = db:select('files', { where = { path = paths } })
+
+        database.__drop_by_path(paths)
+        for _, file in ipairs(files) do
+            local count = db:eval([[SELECT COUNT(1) FROM files WHERE id = :id ]], { id = file.id })
+            assert(count[1]['COUNT(1)'] == 0)
+
+            count = db:eval([[SELECT COUNT(1) FROM inheritance WHERE path_id = :id ]], { id = file.id })
+            assert(count[1]['COUNT(1)'] == 0)
+
+            count = db:eval([[SELECT COUNT(1) FROM definitions WHERE path_id = :id ]], { id = file.id })
+            assert(count[1]['COUNT(1)'] == 0)
+        end
+    end)
 end)

test/parse/inheritance_spec.lua

@@ -17,10 +17,12 @@ describe('parse.inheritance', function()
             local rows = db:eval([[select * from inheritance i where i.base = :name ]], { name = 'Interface1' })
             assert(#rows == 1)
             assert(rows[1].name == 'Class1')
+            assert(rows[1].path_id ~= nil)
 
             rows = db:eval([[select * from inheritance i where i.base = :name ]], { name = 'Attribute' })
             assert(#rows == 1)
             assert(rows[1].name == 'Attributed1Attribute')
+            assert(rows[1].path_id ~= nil)
         end, writer)
     end)
 end)