[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully
This commit is contained in:
tobi
2022-03-07 11:08:26 +01:00
committed by GitHub
parent 100f1280a6
commit 07727753b9
424 changed files with 637100 additions and 176498 deletions

View File

@@ -9,8 +9,6 @@ package windows
import (
errorspkg "errors"
"unsafe"
"golang.org/x/sys/internal/unsafeheader"
)
// EscapeArg rewrites command line argument s as prescribed
@@ -147,8 +145,12 @@ func NewProcThreadAttributeList(maxAttrCount uint32) (*ProcThreadAttributeListCo
}
return nil, err
}
alloc, err := LocalAlloc(LMEM_FIXED, uint32(size))
if err != nil {
return nil, err
}
// size is guaranteed to be ≥1 by InitializeProcThreadAttributeList.
al := &ProcThreadAttributeListContainer{data: (*ProcThreadAttributeList)(unsafe.Pointer(&make([]byte, size)[0]))}
al := &ProcThreadAttributeListContainer{data: (*ProcThreadAttributeList)(unsafe.Pointer(alloc))}
err = initializeProcThreadAttributeList(al.data, maxAttrCount, 0, &size)
if err != nil {
return nil, err
@@ -157,36 +159,17 @@ func NewProcThreadAttributeList(maxAttrCount uint32) (*ProcThreadAttributeListCo
}
// Update modifies the ProcThreadAttributeList using UpdateProcThreadAttribute.
// Note that the value passed to this function will be copied into memory
// allocated by LocalAlloc, the contents of which should not contain any
// Go-managed pointers, even if the passed value itself is a Go-managed
// pointer.
func (al *ProcThreadAttributeListContainer) Update(attribute uintptr, value unsafe.Pointer, size uintptr) error {
alloc, err := LocalAlloc(LMEM_FIXED, uint32(size))
if err != nil {
return err
}
var src, dst []byte
hdr := (*unsafeheader.Slice)(unsafe.Pointer(&src))
hdr.Data = value
hdr.Cap = int(size)
hdr.Len = int(size)
hdr = (*unsafeheader.Slice)(unsafe.Pointer(&dst))
hdr.Data = unsafe.Pointer(alloc)
hdr.Cap = int(size)
hdr.Len = int(size)
copy(dst, src)
al.heapAllocations = append(al.heapAllocations, alloc)
return updateProcThreadAttribute(al.data, 0, attribute, unsafe.Pointer(alloc), size, nil, nil)
al.pointers = append(al.pointers, value)
return updateProcThreadAttribute(al.data, 0, attribute, value, size, nil, nil)
}
// Delete frees ProcThreadAttributeList's resources.
func (al *ProcThreadAttributeListContainer) Delete() {
deleteProcThreadAttributeList(al.data)
for i := range al.heapAllocations {
LocalFree(Handle(al.heapAllocations[i]))
}
al.heapAllocations = nil
LocalFree(Handle(unsafe.Pointer(al.data)))
al.data = nil
al.pointers = nil
}
// List returns the actual ProcThreadAttributeList to be passed to StartupInfoEx.