fix: cannot delete mass files (>=333) in SQLite (#622)

models/file.go

@@ -191,14 +191,15 @@ func RemoveFilesWithSoftLinks(files []File) ([]File, error) {
 	}
 
 	// 查询软链接的文件
-	var filesWithSoftLinks []File
+	filesWithSoftLinks := make([]File, 0)
-	tx := DB
+	for _, file := range files {
-	for _, value := range files {
+		var softLinkFile File
-		tx = tx.Or("source_name = ? and policy_id = ? and id != ?", value.SourceName, value.PolicyID, value.ID)
+		res := DB.
+			Where("source_name = ? and policy_id = ? and id != ?", file.SourceName, file.PolicyID, file.ID).
+			First(&softLinkFile)
+		if res.Error == nil {
+			filesWithSoftLinks = append(filesWithSoftLinks, softLinkFile)
 		}
-	result := tx.Find(&filesWithSoftLinks)
-	if result.Error != nil {
-		return nil, result.Error
 	}
 
 	// 过滤具有软连接的文件

models/file_test.go

@@ -285,30 +285,34 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) {
 		},
 	}
 
+	// 传入空文件列表
+	{
+		file, err := RemoveFilesWithSoftLinks([]File{})
+		asserts.NoError(err)
+		asserts.Empty(file)
+	}
+
 	// 全都没有
 	{
 		mock.ExpectQuery("SELECT(.+)files(.+)").
-			WithArgs("1.txt", 23, 1, "2.txt", 24, 2).
+			WithArgs("1.txt", 23, 1).
+			WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
+		mock.ExpectQuery("SELECT(.+)files(.+)").
+			WithArgs("2.txt", 24, 2).
 			WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
 		file, err := RemoveFilesWithSoftLinks(files)
 		asserts.NoError(mock.ExpectationsWereMet())
 		asserts.NoError(err)
 		asserts.Equal(files, file)
 	}
-	// 查询出错
+
-	{
-		mock.ExpectQuery("SELECT(.+)files(.+)").
-			WithArgs("1.txt", 23, 1, "2.txt", 24, 2).
-			WillReturnError(errors.New("error"))
-		file, err := RemoveFilesWithSoftLinks(files)
-		asserts.NoError(mock.ExpectationsWereMet())
-		asserts.Error(err)
-		asserts.Nil(file)
-	}
 	// 第二个是软链
 	{
 		mock.ExpectQuery("SELECT(.+)files(.+)").
-			WithArgs("1.txt", 23, 1, "2.txt", 24, 2).
+			WithArgs("1.txt", 23, 1).
+			WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
+		mock.ExpectQuery("SELECT(.+)files(.+)").
+			WithArgs("2.txt", 24, 2).
 			WillReturnRows(
 				sqlmock.NewRows([]string{"id", "policy_id", "source_name"}).
 					AddRow(3, 24, "2.txt"),
@@ -318,14 +322,18 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) {
 		asserts.NoError(err)
 		asserts.Equal(files[:1], file)
 	}
+
 	// 第一个是软链
 	{
 		mock.ExpectQuery("SELECT(.+)files(.+)").
-			WithArgs("1.txt", 23, 1, "2.txt", 24, 2).
+			WithArgs("1.txt", 23, 1).
 			WillReturnRows(
 				sqlmock.NewRows([]string{"id", "policy_id", "source_name"}).
 					AddRow(3, 23, "1.txt"),
 			)
+		mock.ExpectQuery("SELECT(.+)files(.+)").
+			WithArgs("2.txt", 24, 2).
+			WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
 		file, err := RemoveFilesWithSoftLinks(files)
 		asserts.NoError(mock.ExpectationsWereMet())
 		asserts.NoError(err)
@@ -334,11 +342,16 @@ func TestRemoveFilesWithSoftLinks(t *testing.T) {
 	// 全部是软链
 	{
 		mock.ExpectQuery("SELECT(.+)files(.+)").
-			WithArgs("1.txt", 23, 1, "2.txt", 24, 2).
+			WithArgs("1.txt", 23, 1).
 			WillReturnRows(
 				sqlmock.NewRows([]string{"id", "policy_id", "source_name"}).
-					AddRow(3, 24, "2.txt").
+					AddRow(3, 23, "1.txt"),
-					AddRow(4, 23, "1.txt"),
+			)
+		mock.ExpectQuery("SELECT(.+)files(.+)").
+			WithArgs("2.txt", 24, 2).
+			WillReturnRows(
+				sqlmock.NewRows([]string{"id", "policy_id", "source_name"}).
+					AddRow(3, 24, "2.txt"),
 			)
 		file, err := RemoveFilesWithSoftLinks(files)
 		asserts.NoError(mock.ExpectationsWereMet())

pkg/filesystem/manage_test.go

@@ -472,6 +472,9 @@ func TestFileSystem_Delete(t *testing.T) {
 					AddRow(4, "1.txt", "1.txt", 365, 1),
 			)
 		mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "name", "source_name", "policy_id", "size"}).AddRow(1, "2.txt", "2.txt", 365, 2))
+		// 两次查询软连接
+		mock.ExpectQuery("SELECT(.+)files(.+)").
+			WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
 		mock.ExpectQuery("SELECT(.+)files(.+)").
 			WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
 		// 查询上传策略
@@ -527,6 +530,9 @@ func TestFileSystem_Delete(t *testing.T) {
 					AddRow(4, "1.txt", "1.txt", 602, 1),
 			)
 		mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "name", "source_name", "policy_id", "size"}).AddRow(1, "2.txt", "2.txt", 602, 2))
+		// 两次查询软连接
+		mock.ExpectQuery("SELECT(.+)files(.+)").
+			WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
 		mock.ExpectQuery("SELECT(.+)files(.+)").
 			WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
 		// 查询上传策略