diff --git a/PublishHelperBot/Environment.fs b/PublishHelperBot/Environment.fs index 44497bb..0f3ec47 100644 --- a/PublishHelperBot/Environment.fs +++ b/PublishHelperBot/Environment.fs @@ -9,6 +9,7 @@ type public BotConfig = { relayUrl: string chanelId: int64 adminChatId: int64 + YoutubeDlUrl: string } let private ReadConfig = diff --git a/PublishHelperBot/Handlers.fs b/PublishHelperBot/Handlers.fs index a708311..a5c020d 100644 --- a/PublishHelperBot/Handlers.fs +++ b/PublishHelperBot/Handlers.fs @@ -1,7 +1,9 @@ module PublishHelperBot.Handlers open System.Threading.Tasks +open Microsoft.FSharp.Core open PublishHelperBot.Environment +open PublishHelperBot.YoutubeDl open Telegram.Bot open Telegram.Bot.Types open Telegram.Bot.Types.Enums @@ -10,6 +12,7 @@ type BaseHandlerArgs = Update * BotConfig type HandlerArgs = Update * BotConfig * ITelegramBotClient type HandlerRequirements = BaseHandlerArgs -> bool type Handler = HandlerArgs -> Task +type Handler<'deps> = 'deps * HandlerArgs -> Task // Utils let UpdateIsAMessage (x: Update) = x.Type = UpdateType.Message @@ -67,11 +70,20 @@ let public RelayHandler: Handler = fun (u, c, tg) -> | _ -> Task.CompletedTask -// Youtube repost +// YoutubeDL repost +let YoutubeRepostMatchCmd = "\\ytdl" let public YoutubeRepostMatch: HandlerRequirements = fun (u, c) -> UpdateIsAMessage u && FromAdminChat <| (u.Message, c) && - u.Message.Text.StartsWith("\\yt") && + u.Message.Text.StartsWith YoutubeRepostMatchCmd && u.Message.Text.Split(' ').Length = 2 + +let public YoutubeRepostHandler: Handler = fun (yt, (u, c, tg)) -> + task { + let trim (x: string) = x.Trim() + let! id = YoutubeRepostMatchCmd |> u.Message.Text.Split |> Array.last |> trim |> yt.EnqueueJob + do! tg.SendTextMessageAsync(c.adminChatId, id.ToString()) |> Async.AwaitTask |> Async.Ignore + } + \ No newline at end of file diff --git a/PublishHelperBot/Program.fs b/PublishHelperBot/Program.fs index b25c6a1..9e4d39e 100644 --- a/PublishHelperBot/Program.fs +++ b/PublishHelperBot/Program.fs @@ -6,6 +6,7 @@ open System.Threading open System.Threading.Tasks open PublishHelperBot.Handlers open PublishHelperBot.Environment +open PublishHelperBot.YoutubeDl open Telegram.Bot open Telegram.Bot.Polling open Telegram.Bot.Types @@ -14,13 +15,18 @@ open Telegram.Bot.Types.Enums let CreateBot (config: BotConfig, http: HttpClient) = TelegramBotClient(config.token, http) let config = CreateConfig <| "SBPB_CONFIG_PATH"; let botClient = CreateBot <| (config, new HttpClient()) +let YtService = YoutubeDlBackgroundService <| + (new HttpClient(), config.YoutubeDlUrl, botClient, config.chanelId, CancellationToken.None) + let startDate = DateTime.UtcNow let isObsoleteUpdate (u: Update) = u.Type = UpdateType.Message && u.Message.Date < startDate; let updateHandle (bc: ITelegramBotClient) (u: Update) (ct: CancellationToken): Task = + let tgCtx = (u, config, bc) match u with | _ when isObsoleteUpdate u -> Task.CompletedTask - | _ when RelayMatch <| (u, config) -> RelayHandler <| (u, config, bc) + | _ when RelayMatch <| (u, config) -> RelayHandler <| tgCtx + | _ when YoutubeRepostMatch <| (u, config) -> YoutubeRepostHandler <| (YtService, tgCtx) | _ -> Task.CompletedTask let handlePollingError (bc: ITelegramBotClient) (e: Exception) (t: CancellationToken) = @@ -30,6 +36,7 @@ let handlePollingError (bc: ITelegramBotClient) (e: Exception) (t: CancellationT let receiverOptions = ReceiverOptions(AllowedUpdates = Array.zeroCreate 0) botClient.StartReceiving(updateHandle,handlePollingError,receiverOptions) +YtService.StartYoutubeDlService() printf "Я родился" Console.ReadKey() |> ignore \ No newline at end of file diff --git a/PublishHelperBot/PublishHelperBot.fsproj b/PublishHelperBot/PublishHelperBot.fsproj index 2176efd..78fdc6f 100644 --- a/PublishHelperBot/PublishHelperBot.fsproj +++ b/PublishHelperBot/PublishHelperBot.fsproj @@ -7,10 +7,12 @@ + + diff --git a/PublishHelperBot/YoutubeDl.fs b/PublishHelperBot/YoutubeDl.fs new file mode 100644 index 0000000..a9023a3 --- /dev/null +++ b/PublishHelperBot/YoutubeDl.fs @@ -0,0 +1,176 @@ +module PublishHelperBot.YoutubeDl + +open System +open System.Collections.Generic +open System.IO +open System.Net.Http +open System.Text +open System.Threading +open Microsoft.FSharp.Core +open Newtonsoft.Json +open Nito.AsyncEx +open Telegram.Bot +open Telegram.Bot.Types.InputFiles + +type public CreateYoutubeDLUrl = string +type public ChatId = int64 +type public CreateYoutubeDLJob = { + url: string + savePath: string +} +type public CreateYoutubeDLJobSuccess = { + task: Guid +} + +type public YoutubeDlStateResponse = { + state: string +} + +type public YoutubeDlError = { + message: string +} +type public YoutubeDlJob<'A> = { + internalId: Guid + externalId: 'A + url: string + state: string + savePath: string +} + +type YoutubeDlClientActions = Create | Check | Delete +type HttpMethods = GET | POST | DELETE +type CreateJobResult = Result Async +type CheckJobResult = Result Async +type CleanJobResult = Result Async +type StartYoutubeDlServiceArgs = HttpClient * CreateYoutubeDLUrl * ITelegramBotClient * ChatId * CancellationToken +type YoutubeDlJobWithId = YoutubeDlJob +type YoutubeDlJobWithoutId = YoutubeDlJob +type YoutubeDlCurrentJob = + | Created of YoutubeDlJobWithoutId + | Awaiting of YoutubeDlJobWithId + | Downloaded of YoutubeDlJobWithId + | Done of YoutubeDlJobWithId + | None of unit + +let inline () (lck: AsyncLock) f = async { + use! __ = Async.AwaitTask <| lck.LockAsync().AsTask() + return! f +} + +type YoutubeDlClient(baseUrl: string, client: HttpClient) = + let lock = AsyncLock() + let apiPrefix = $"{baseUrl}api/"; + let ResolvePath(action: YoutubeDlClientActions) = + match action with + | Create -> $"{apiPrefix}download" + | Check -> $"{apiPrefix}status" + | Delete -> $"{apiPrefix}clear" + + let doHttp (url: string, method: HttpMethods, content: HttpContent): Result<'TRes, YoutubeDlError> Async = async { + try + let! res = + match method with + | POST -> client.PostAsync(url, content) |> Async.AwaitTask + | GET -> client.GetAsync(url) |> Async.AwaitTask + | DELETE -> client.DeleteAsync(url) |> Async.AwaitTask + + let! content = res.Content.ReadAsStringAsync() |> Async.AwaitTask + return + match res.IsSuccessStatusCode with + | true -> Ok (JsonConvert.DeserializeObject<'TRes> <| content) + | false -> Error { message = "Unknown network error" } + with + | ex -> return Error { message = ex.Message } + } + + member this.CreateJob(model: CreateYoutubeDLJob): CreateJobResult = lock async { + use content = new StringContent(JsonConvert.SerializeObject <| model, Encoding.UTF8, "application/json") + return! doHttp <| (ResolvePath Create, POST, content) + } + + member this.CheckJob(id: Guid): CheckJobResult = lock async { + let arg = [KeyValuePair("id", id.ToString())] + use content = new FormUrlEncodedContent(arg) + let! query = content.ReadAsStringAsync() |> Async.AwaitTask + return! doHttp <| ($"{ResolvePath Check}?{query}", GET, content) + } + + member this.CleanJob(id: Guid): CleanJobResult = lock async { + let arg = [KeyValuePair("id", id.ToString())] + use content = new FormUrlEncodedContent(arg) + let! query = content.ReadAsStringAsync() |> Async.AwaitTask + return! doHttp <| ($"{ResolvePath Delete}?{query}", DELETE, content) + } + +type YoutubeDlBackgroundService(requirements: StartYoutubeDlServiceArgs) = + let (http, url, tg, chatId, ct) = requirements + let lock = AsyncLock() + let mutable currentJob: YoutubeDlCurrentJob = None () + let jobPool = Queue() + let ytClient = YoutubeDlClient <| (url, http) + let mapJobToApi (job: YoutubeDlJob<_>): CreateYoutubeDLJob = { + url = job.url + savePath = job.savePath + } + let attachExternalId (id: Guid, job: YoutubeDlJobWithoutId): YoutubeDlCurrentJob = + Awaiting { internalId = job.internalId; state = job.state; url = job.state; externalId = id; savePath = job.savePath } + + let tryAssignNewJob() = async { + let (result, job) = jobPool.TryDequeue() + match result with + | true -> currentJob <- Created job + | false -> currentJob <- None () + } + + let uploadToYtDl(job: YoutubeDlJobWithoutId) = async { + match! ytClient.CreateJob <| mapJobToApi job with + | Ok x -> currentJob <- attachExternalId <| (x.task, job) + // TODO: Logging! + | Error _ -> currentJob <- None () + } + + let checkJob(job: YoutubeDlJobWithId) = async { + match! ytClient.CheckJob <| job.externalId with + | Ok x when x.state.Equals("Finished", StringComparison.OrdinalIgnoreCase) -> currentJob <- Downloaded job + | Error _ -> currentJob <- None () + | _ -> () + // That's take a while + do! Async.Sleep 5000 + } + + let postVideo(job: YoutubeDlJobWithId) = async { + use file = File.OpenRead <| job.savePath + let input = InputOnlineFile(file, job.savePath) + let caption = $"Source: {job.url}" + do! tg.SendVideoAsync(chatId, input, caption = caption) |> Async.AwaitTask |> Async.Ignore + currentJob <- Done job + } + + let cleanUp(job: YoutubeDlJobWithId) = async { + match! ytClient.CleanJob <| job.externalId with + | Ok _ -> currentJob <- None () + | Error _ -> currentJob <- None () + } + + let chooseAction() = lock async { + do! match currentJob with + | None _ -> tryAssignNewJob() + | Created x -> x |> uploadToYtDl + | Awaiting x -> x |> checkJob + | Downloaded x -> x |> postVideo + | Done x -> x |> cleanUp + } + + let rec loop () = async { + do! match ct.IsCancellationRequested with + | false -> chooseAction() + | true -> async { () } + do! Async.Sleep 150 + return! loop() + } + member public this.StartYoutubeDlService() = loop() |> Async.Start + member public this.EnqueueJob(url: string) = lock async { + let id = Guid.NewGuid() + jobPool.Enqueue({ internalId = id; externalId = (); state = "new"; url = url; savePath = Path.GetTempFileName() }) + return id + } \ No newline at end of file diff --git a/youtube-dl-api/.gitignore b/youtube-dl-api/.gitignore new file mode 100644 index 0000000..46926dc --- /dev/null +++ b/youtube-dl-api/.gitignore @@ -0,0 +1,286 @@ +# Created by https://www.toptal.com/developers/gitignore/api/pycharm,python +# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm,python + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +# Azure Toolkit for IntelliJ plugin +# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij +.idea/**/azureSettings.xml + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# End of https://www.toptal.com/developers/gitignore/api/pycharm,python \ No newline at end of file diff --git a/youtube-dl-api/.idea/.gitignore b/youtube-dl-api/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/youtube-dl-api/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/youtube-dl-api/.idea/inspectionProfiles/profiles_settings.xml b/youtube-dl-api/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/youtube-dl-api/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/youtube-dl-api/.idea/misc.xml b/youtube-dl-api/.idea/misc.xml new file mode 100644 index 0000000..609ee93 --- /dev/null +++ b/youtube-dl-api/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/youtube-dl-api/.idea/modules.xml b/youtube-dl-api/.idea/modules.xml new file mode 100644 index 0000000..cbc0090 --- /dev/null +++ b/youtube-dl-api/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/youtube-dl-api/.idea/vcs.xml b/youtube-dl-api/.idea/vcs.xml new file mode 100644 index 0000000..6c0b863 --- /dev/null +++ b/youtube-dl-api/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/youtube-dl-api/.idea/youtube-dl-api.iml b/youtube-dl-api/.idea/youtube-dl-api.iml new file mode 100644 index 0000000..d0876a7 --- /dev/null +++ b/youtube-dl-api/.idea/youtube-dl-api.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/youtube-dl-api/README.md b/youtube-dl-api/README.md new file mode 100644 index 0000000..e69de29 diff --git a/youtube-dl-api/poetry.lock b/youtube-dl-api/poetry.lock new file mode 100644 index 0000000..04971a5 --- /dev/null +++ b/youtube-dl-api/poetry.lock @@ -0,0 +1,226 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + +[[package]] +name = "anyio" +version = "3.6.2" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" +optional = false +python-versions = ">=3.6.2" +files = [ + {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"}, + {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"}, +] + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] +trio = ["trio (>=0.16,<0.22)"] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "fastapi" +version = "0.89.1" +description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "fastapi-0.89.1-py3-none-any.whl", hash = "sha256:f9773ea22290635b2f48b4275b2bf69a8fa721fda2e38228bed47139839dc877"}, + {file = "fastapi-0.89.1.tar.gz", hash = "sha256:15d9271ee52b572a015ca2ae5c72e1ce4241dd8532a534ad4f7ec70c376a580f"}, +] + +[package.dependencies] +pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0" +starlette = "0.22.0" + +[package.extras] +all = ["email-validator (>=1.1.1)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "python-multipart (>=0.0.5)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] +dev = ["pre-commit (>=2.17.0,<3.0.0)", "ruff (==0.0.138)", "uvicorn[standard] (>=0.12.0,<0.21.0)"] +doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pyyaml (>=5.3.1,<7.0.0)", "typer[all] (>=0.6.1,<0.8.0)"] +test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==22.10.0)", "coverage[toml] (>=6.5.0,<8.0)", "databases[sqlite] (>=0.3.2,<0.7.0)", "email-validator (>=1.1.1,<2.0.0)", "flask (>=1.1.2,<3.0.0)", "httpx (>=0.23.0,<0.24.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.982)", "orjson (>=3.2.1,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest (>=7.1.3,<8.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<7.0.0)", "ruff (==0.0.138)", "sqlalchemy (>=1.3.18,<1.4.43)", "types-orjson (==3.6.2)", "types-ujson (==5.6.0.0)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)"] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "pydantic" +version = "1.10.4" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, + {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"}, + {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"}, + {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"}, + {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"}, + {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"}, + {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"}, + {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"}, + {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, + {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + +[[package]] +name = "starlette" +version = "0.22.0" +description = "The little ASGI library that shines." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "starlette-0.22.0-py3-none-any.whl", hash = "sha256:b5eda991ad5f0ee5d8ce4c4540202a573bb6691ecd0c712262d0bc85cf8f2c50"}, + {file = "starlette-0.22.0.tar.gz", hash = "sha256:b092cbc365bea34dd6840b42861bdabb2f507f8671e642e8272d2442e08ea4ff"}, +] + +[package.dependencies] +anyio = ">=3.4.0,<5" + +[package.extras] +full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] + +[[package]] +name = "uvicorn" +version = "0.20.0" +description = "The lightning-fast ASGI server." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "uvicorn-0.20.0-py3-none-any.whl", hash = "sha256:c3ed1598a5668208723f2bb49336f4509424ad198d6ab2615b7783db58d919fd"}, + {file = "uvicorn-0.20.0.tar.gz", hash = "sha256:a4e12017b940247f836bc90b72e725d7dfd0c8ed1c51eb365f5ba30d9f5127d8"}, +] + +[package.dependencies] +click = ">=7.0" +h11 = ">=0.8" + +[package.extras] +standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] + +[[package]] +name = "youtube-dl" +version = "2021.12.17" +description = "YouTube video downloader" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "youtube_dl-2021.12.17-py2.py3-none-any.whl", hash = "sha256:f1336d5de68647e0364a47b3c0712578e59ec76f02048ff5c50ef1c69d79cd55"}, + {file = "youtube_dl-2021.12.17.tar.gz", hash = "sha256:bc59e86c5d15d887ac590454511f08ce2c47698d5a82c27bfe27b5d814bbaed2"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "51ec164c892a17017f004b82d07fc561a863fbe4859baff2482a44b5776f6494" diff --git a/youtube-dl-api/pyproject.toml b/youtube-dl-api/pyproject.toml new file mode 100644 index 0000000..acac1da --- /dev/null +++ b/youtube-dl-api/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "youtube-dl-api" +version = "0.1.0" +description = "" +authors = ["Keroosha "] +readme = "README.md" +packages = [{include = "youtube_dl_api"}] + +[tool.poetry.dependencies] +python = "3.10.9" +youtube_dl = "2021.12.17" +fastapi = "0.89.1" +uvicorn = "0.20.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/youtube-dl-api/tests/__init__.py b/youtube-dl-api/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/youtube-dl-api/youtube_dl_api/__init__.py b/youtube-dl-api/youtube_dl_api/__init__.py new file mode 100644 index 0000000..3524630 --- /dev/null +++ b/youtube-dl-api/youtube_dl_api/__init__.py @@ -0,0 +1,61 @@ +from fastapi import FastAPI, BackgroundTasks, Response +from youtube_dl import YoutubeDL +from uuid import uuid4, UUID +from pydantic import BaseModel + +backgroundJobs = {} +app = FastAPI() + + +class DownloadModel(BaseModel): + url: str + savePath: str + + +def report_state(id: str): + def update(d): + backgroundJobs[id]["state"] = d["status"] + backgroundJobs[id]["_youtube-dl_"] = d + return update + + +def load_video(url: str, file_path: str, id: str): + opts = { + "forcefilename": file_path, + "progress_hooks": [report_state(id)] + } + with YoutubeDL(opts) as ydl: + ydl.download([url]) + + +@app.get("/api/info") +def info(url: str): + with YoutubeDL({}) as ydl: + return ydl.extract_info(url, False) + + +@app.post("/api/download") +def download(background_tasks: BackgroundTasks, model: DownloadModel): + id = uuid4() + backgroundJobs[id] = {"state": "created", "url": model.url, "savePath": model.savePath} + background_tasks.add_task(load_video, model.url, model.savePath, id) + return {"task": id} + + +@app.get("/api/status") +def status(response: Response, id: UUID): + if id in backgroundJobs.keys(): + return backgroundJobs[id] + + response.status_code = 404 + return {"state": "not found"} + + +@app.delete("/api/clear") +def clear(response: Response, id: UUID): + if id in backgroundJobs.keys(): + del backgroundJobs[id] + return {"state": "done"} + + response.status_code = 404 + return {"state": "not found"}