malhuda/ab-download-manager
1
0
Fork 0
mirror of https://github.com/amir1376/ab-download-manager.git synced 2025-02-20 11:43:24 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add sparse file creation mode

Browse Source
This commit is contained in:
AmirHossein Abdolmotallebi 2024-08-29 21:57:04 +03:30
parent 36ea671893
commit 5472d9bef5
11 changed files with 261 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
desktop/app/src/main/kotlin/com/abdownloadmanager/desktop/di/Di.kt
View File

@ -35,6 +35,7 @@ import org.koin.dsl.module
import com.abdownloadmanager.updatechecker.DummyUpdateChecker
import com.abdownloadmanager.updatechecker.UpdateChecker
import ir.amirab.downloader.monitor.IDownloadMonitor
import ir.amirab.downloader.utils.EmptyFileCreator
val downloaderModule = module {
single<IDownloadQueueDatabase> {
@ -90,9 +91,15 @@ val downloaderModule = module {
)
}
single {
DownloadManager(get(), get(), get(), get(), get())
val downloadSettings:DownloadSettings = get()
EmptyFileCreator(
diskStat = get(),
useSparseFile = { downloadSettings.useSparseFileAllocation }
)
}
single {
DownloadManager(get(), get(), get(), get(), get(), get())
}.bind(DownloadManagerMinimalControl::class)
single<IDownloadMonitor> {
DownloadMonitor(get())

18
desktop/app/src/main/kotlin/com/abdownloadmanager/desktop/pages/settings/SettingsComponent.kt
View File

@ -77,6 +77,21 @@ fun useServerLastModified(appRepository: AppRepository): BooleanConfigurable {
)
}
fun useSparseFileAllocation(appRepository: AppRepository): BooleanConfigurable {
return BooleanConfigurable(
title = "Sparse File Allocation",
description = "Create files more efficiently, especially on SSDs, by reducing unnecessary data writing. This can speed up download starts and save disk space. If downloads start slowly, consider disabling this option, as it may not be properly supported on some devices.",
backedBy = appRepository.useSparseFileAllocation,
describe = {
if (it) {
"Enabled"
} else {
"Disabled"
}
},
)
}
fun speedLimitConfig(appRepository: AppRepository): SpeedLimitConfigurable {
return SpeedLimitConfigurable(
title = "Global Speed Limiter",
@ -263,7 +278,8 @@ class SettingsComponent(
speedLimitConfig(appRepository),
threadCountConfig(appRepository),
dynamicPartDownloadConfig(appRepository),
useServerLastModified(appRepository)
useServerLastModified(appRepository),
useSparseFileAllocation(appRepository)
)
}
}

7
desktop/app/src/main/kotlin/com/abdownloadmanager/desktop/repository/AppRepository.kt
View File

@ -32,6 +32,7 @@ class AppRepository : KoinComponent {
val threadCount = appSettings.threadCount
val dynamicPartCreation = appSettings.dynamicPartCreation
val useServerLastModifiedTime = appSettings.useServerLastModifiedTime
val useSparseFileAllocation = appSettings.useSparseFileAllocation
val useAverageSpeed = appSettings.useAverageSpeed
val saveLocation = appSettings.defaultDownloadFolder
val integrationEnabled = appSettings.browserIntegrationEnabled
@ -73,6 +74,12 @@ class AppRepository : KoinComponent {
downloadSettings.useServerLastModifiedTime = it
downloadManager.reloadSetting()
}.launchIn(scope)
useSparseFileAllocation
.debounce(500)
.onEach {
downloadSettings.useSparseFileAllocation = it
downloadManager.reloadSetting()
}.launchIn(scope)
integrationPort
.debounce(500)
.onEach {

5
desktop/app/src/main/kotlin/com/abdownloadmanager/desktop/storage/AppSettingsStorage.kt
View File

@ -19,6 +19,7 @@ data class AppSettingsModel(
val threadCount: Int = 5,
val dynamicPartCreation: Boolean = true,
val useServerLastModifiedTime: Boolean = false,
val useSparseFileAllocation: Boolean = true,
val useAverageSpeed: Boolean = true,
val speedLimit: Long = 0,
val autoStartOnBoot: Boolean = true,
@ -39,6 +40,7 @@ data class AppSettingsModel(
val threadCount = intKeyOf("threadCount")
val dynamicPartCreation = booleanKeyOf("dynamicPartCreation")
val useServerLastModifiedTime = booleanKeyOf("useServerLastModifiedTime")
val useSparseFileAllocation = booleanKeyOf("useSparseFileAllocation")
val useAverageSpeed = booleanKeyOf("useAverageSpeed")
val speedLimit = longKeyOf("speedLimit")
val autoStartOnBoot = booleanKeyOf("autoStartOnBoot")
@ -57,6 +59,7 @@ data class AppSettingsModel(
threadCount = source.get(Keys.threadCount) ?: default.threadCount,
dynamicPartCreation = source.get(Keys.dynamicPartCreation) ?: default.dynamicPartCreation,
useServerLastModifiedTime = source.get(Keys.useServerLastModifiedTime) ?: default.useServerLastModifiedTime,
useSparseFileAllocation = source.get(Keys.useSparseFileAllocation) ?: default.useSparseFileAllocation,
useAverageSpeed = source.get(Keys.useAverageSpeed) ?: default.useAverageSpeed,
speedLimit = source.get(Keys.speedLimit) ?: default.speedLimit,
autoStartOnBoot = source.get(Keys.autoStartOnBoot) ?: default.autoStartOnBoot,
@ -74,6 +77,7 @@ data class AppSettingsModel(
put(Keys.threadCount, focus.threadCount)
put(Keys.dynamicPartCreation, focus.dynamicPartCreation)
put(Keys.useServerLastModifiedTime, focus.useServerLastModifiedTime)
put(Keys.useSparseFileAllocation, focus.useSparseFileAllocation)
put(Keys.useAverageSpeed, focus.useAverageSpeed)
put(Keys.speedLimit, focus.speedLimit)
put(Keys.autoStartOnBoot, focus.autoStartOnBoot)
@ -93,6 +97,7 @@ class AppSettingsStorage(
val threadCount = from(AppSettingsModel.threadCount)
val dynamicPartCreation = from(AppSettingsModel.dynamicPartCreation)
val useServerLastModifiedTime = from(AppSettingsModel.useServerLastModifiedTime)
val useSparseFileAllocation = from(AppSettingsModel.useSparseFileAllocation)
val useAverageSpeed = from(AppSettingsModel.useAverageSpeed)
val speedLimit = from(AppSettingsModel.speedLimit)
val autoStartOnBoot = from(AppSettingsModel.autoStartOnBoot)

2
downloader/core/src/main/kotlin/ir/amirab/downloader/DownloadManager.kt
View File

@ -7,6 +7,7 @@ import ir.amirab.downloader.downloaditem.*
import ir.amirab.downloader.downloaditem.contexts.DuplicateRemoval
import ir.amirab.downloader.downloaditem.contexts.RemovedBy
import ir.amirab.downloader.part.Part
import ir.amirab.downloader.utils.EmptyFileCreator
import ir.amirab.downloader.utils.FileNameUtil
import ir.amirab.downloader.utils.IDiskStat
import ir.amirab.downloader.utils.OnDuplicateStrategy
@ -30,6 +31,7 @@ class DownloadManager(
val partListDb: IDownloadPartListDb,
val settings: DownloadSettings,
val diskStat: IDiskStat,
val emptyFileCreator: EmptyFileCreator,
val client: DownloaderClient,
) : DownloadManagerMinimalControl {

1
downloader/core/src/main/kotlin/ir/amirab/downloader/DownloadSettings.kt
View File

@ -6,5 +6,6 @@ data class DownloadSettings(
var dynamicPartCreationMode: Boolean = true,
var useServerLastModifiedTime: Boolean = false,
var globalSpeedLimit: Long = 0,//unlimited
var useSparseFileAllocation: Boolean = true,
val minPartSize: Long = 2048,//2kB
)

2
downloader/core/src/main/kotlin/ir/amirab/downloader/destination/DownloadDestination.kt
View File

@ -57,7 +57,7 @@ abstract class DownloadDestination(
outputFile.delete()
}
abstract suspend fun prepareFile(onProgressUpdate: (Int) -> Unit)
abstract suspend fun prepareFile(onProgressUpdate: (Int?) -> Unit)
abstract suspend fun isDownloadedPartsIsValid(): Boolean
abstract fun flush()
open fun onPartCancelled(part: Part){

40
downloader/core/src/main/kotlin/ir/amirab/downloader/destination/SimpleDownloadDestination.kt
View File

@ -1,8 +1,10 @@
package ir.amirab.downloader.destination
import ir.amirab.downloader.DownloadSettings
import ir.amirab.downloader.anntation.HeavyCall
import ir.amirab.downloader.exception.NoSpaceInStorageException
import ir.amirab.downloader.part.Part
import ir.amirab.downloader.utils.EmptyFileCreator
import ir.amirab.downloader.utils.IDiskStat
import ir.amirab.downloader.utils.calcPercent
import kotlinx.coroutines.Dispatchers
@ -13,11 +15,11 @@ import okio.FileSystem
import okio.Path.Companion.toOkioPath
import java.io.File
import java.io.FileOutputStream
import java.io.RandomAccessFile
class SimpleDownloadDestination(
file: File,
private val diskStat: IDiskStat,
private val emptyFileCreator: EmptyFileCreator,
) : DownloadDestination(file) {
private var _fileHandle: FileHandle? = null
@ -83,7 +85,7 @@ class SimpleDownloadDestination(
}
@HeavyCall
override suspend fun prepareFile(onProgressUpdate: (Int) -> Unit) {
override suspend fun prepareFile(onProgressUpdate: (Int?) -> Unit) {
// println("preparing file ")
// println("file info path=$outputFile size=${outputFile.runCatching { length() }.getOrNull()}")
outputFile.parentFile.let {
@ -96,38 +98,8 @@ class SimpleDownloadDestination(
error("${outputFile.parentFile} is not a directory")
}
}
onProgressUpdate(0)
if (!outputFile.exists()) {
// println("file not exist creating...")
outputFile.createNewFile()
}
// println("file size before modification ${outputFile.length()}")
// println("currentFileSize: " + outputFile.length())
if (outputSize==-1L){
withContext(Dispatchers.IO){
RandomAccessFile(outputFile, "rw").use {
it.channel.use {
//clear
it.truncate(0)
}
}
}
}else if (outputFile.length() > outputSize) {
// println("current file size bigger that output size truncating fSize:${outputFile.length()} oSize:${outputSize}")
withContext(Dispatchers.IO) {
RandomAccessFile(outputFile, "rw").use {
it.channel.use {
it.truncate(outputSize)
}
}
}
onProgressUpdate(100)
} else if (outputFile.length() < outputSize) {
// println("current file size smaller than output size filling ${outputFile.length()} to reach ${outputSize}")
// println("filling output")
fillOutput(onProgressUpdate)
}
// println("length of prepared file ${outputFile.length()}")
emptyFileCreator
.prepareFile(outputFile, outputSize,onProgressUpdate)
}
/**

15
downloader/core/src/main/kotlin/ir/amirab/downloader/downloaditem/DownloadJob.kt
View File

@ -10,10 +10,7 @@ import ir.amirab.downloader.exception.DownloadValidationException
import ir.amirab.downloader.exception.FileChangedException
import ir.amirab.downloader.exception.TooManyErrorException
import ir.amirab.downloader.part.*
import ir.amirab.downloader.utils.ExceptionUtils
import ir.amirab.downloader.utils.TimeUtils
import ir.amirab.downloader.utils.printStackIfNOtUsual
import ir.amirab.downloader.utils.splitToRange
import ir.amirab.downloader.utils.*
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.asStateFlow
@ -59,7 +56,11 @@ class DownloadJob(
suspend fun boot() {
if (!booted) {
val outFile = downloadManager.calculateOutputFile(downloadItem)
destination = SimpleDownloadDestination(outFile, downloadManager.diskStat)
destination = SimpleDownloadDestination(
file = outFile,
diskStat = downloadManager.diskStat,
emptyFileCreator = downloadManager.emptyFileCreator
)
loadPartState()
supportsConcurrent = when (getParts().size) {
in 2..Int.MAX_VALUE -> true
@ -172,7 +173,7 @@ class DownloadJob(
private suspend fun prepareDestination(
onProgressUpdate: (Int) -> Unit,
onProgressUpdate: (Int?) -> Unit,
) {
withContext(Dispatchers.IO) {
destination.outputSize = downloadItem.contentLength
@ -655,7 +656,7 @@ sealed class DownloadJobStatus(
data object Resuming : DownloadJobStatus(0, DownloadStatus.Downloading),
IsActive
data class PreparingFile(val percent: Int) : DownloadJobStatus(1, DownloadStatus.Downloading),
data class PreparingFile(val percent: Int?) : DownloadJobStatus(1, DownloadStatus.Downloading),
IsActive
data class Canceled(val e: Throwable) : DownloadJobStatus(

137
downloader/core/src/main/kotlin/ir/amirab/downloader/utils/EmptyFileCreator.kt Normal file
View File

@ -0,0 +1,137 @@
package ir.amirab.downloader.utils
import ir.amirab.downloader.exception.NoSpaceInStorageException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.isActive
import kotlinx.coroutines.withContext
import java.io.File
import java.io.FileOutputStream
import java.io.IOException
import java.io.RandomAccessFile
class EmptyFileCreator(
private val diskStat: IDiskStat,
private val useSparseFile: () -> Boolean
) {
private fun canWeUseSparse(file: File): Boolean {
return useSparseFile() && SparseFile.canWeCreateSparseFile(file)
}
/**
* @param length must be -1 , or positive
*/
suspend fun prepareFile(
file: File,
length: Long,
onProgressUpdate: (percent: Int?) -> Unit,
) {
require(length >= -1) {
"length must be -1 , or positive value but we got ${length}"
}
withContext(Dispatchers.IO) {
val canWeUseSparse = canWeUseSparse(file)
onProgressUpdate(0)
if (length == -1L) {
RandomAccessFile(file, "rw").use {
it.setLength(0)
}
onProgressUpdate(100)
return@withContext
}
val remainingSpace = diskStat.getRemainingSpace(file.parentFile)
if (file.exists()) {
val currentLength = file.length()
val requiredLength = length - currentLength
if (remainingSpace < requiredLength) {
throw NoSpaceInStorageException(remainingSpace, requiredLength)
}
when {
currentLength > length -> {
RandomAccessFile(file, "rw").use {
it.setLength(length)
}
onProgressUpdate(100)
return@withContext
}
currentLength < length -> {
if (canWeUseSparse) {
if (!file.delete()) {
throw IOException("can't delete file")
}
if (SparseFile.createSparseFile(file)) {
onProgressUpdate(null)
writeAtLast(file, length)
} else {
file.createNewFile()
fillOutput(file, length, onProgressUpdate)
}
} else {
fillOutput(file, length, onProgressUpdate)
}
onProgressUpdate(100)
return@withContext
}
else -> {
onProgressUpdate(100)
return@withContext
}
}
} else {
if (remainingSpace < length) {
throw NoSpaceInStorageException(remainingSpace, length)
}
if (canWeUseSparse && SparseFile.createSparseFile(file)) {
onProgressUpdate(null)
writeAtLast(file, length)
} else {
file.createNewFile()
fillOutput(file, length, onProgressUpdate)
}
onProgressUpdate(100)
}
}
}
/**
* manually write a single byte to the last of file!
* if the sparse is not supported for the file, at least
* waits for OS to create empty file for us
*/
private fun writeAtLast(file: File, length: Long) {
RandomAccessFile(file, "rw").use {
it.seek(length - 1)
it.write(0)
}
}
private suspend fun fillOutput(outputFile: File, length: Long, onProgressUpdate: (percent: Int) -> Unit) {
val much = length - outputFile.length()
val remainingSpace = diskStat.getRemainingSpace(outputFile.parentFile)
if (remainingSpace < much) {
throw NoSpaceInStorageException(remainingSpace, much)
}
// println("how much to be appended $much")
withContext(Dispatchers.IO) {
FileOutputStream(outputFile, true).use {
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
var written = 0L
while (isActive) {
val writeInThisLoop = if (much - written > buffer.size) {
buffer.size
} else {
much - written
}.toInt()
if (writeInThisLoop == 0) break
// println(writeInThisLoop)
it.write(buffer, 0, writeInThisLoop)
written += writeInThisLoop
onProgressUpdate(calcPercent(written, much))
}
}
}
}
}

68
downloader/core/src/main/kotlin/ir/amirab/downloader/utils/SparseFile.kt Normal file
View File

@ -0,0 +1,68 @@
package ir.amirab.downloader.utils
import java.io.File
import java.nio.file.Files
import java.nio.file.OpenOption
import java.nio.file.StandardOpenOption
import kotlin.io.path.fileStore
object SparseFile {
private val fileSystemsSupportingSparseFiles = listOf(
"NTFS",
"ext4",
"ext3",
"XFS",
"Btrfs",
"ZFS",
"ReiserFS",
"APFS",
"exFAT",
"HFS+",
"UFS",
"ReFS"
)
fun createSparseFile(file: File): Boolean {
if (!file.exists()) {
val options = arrayOf<OpenOption>(
StandardOpenOption.WRITE,
StandardOpenOption.CREATE_NEW,
StandardOpenOption.SPARSE
)
Files.newByteChannel(file.toPath(), *options).use {
return true
}
}
return false
}
/**
* I assume that its parent are created before so make sure of that
*/
fun canWeCreateSparseFile(file: File): Boolean {
return kotlin.runCatching {
val nearestFileExist = file.findNearestExistingFile()?:return false
val type = nearestFileExist
.toPath()
.fileStore()
.type()
fileSystemsSupportingSparseFiles
.find { it.equals(type, true) } != null
}.getOrElse { false }
}
private fun File.findNearestExistingFile(): File? {
var f:File? = this
while (true){
if (f==null){
return null
}
if (f.exists()){
return f
}else{
f = f.parentFile
}
}
}
}