From 0e744af73f7fd086713b4246f4499d58f4814e12 Mon Sep 17 00:00:00 2001 From: Kenneth Date: Thu, 25 Jul 2024 10:36:24 +0100 Subject: [PATCH] feat: initial code for fal.ai integration --- fal_app.py | 35 +++++++++++++++++++++++++++++++++++ generate.py | 4 +++- generate_manual.py | 11 +++-------- prompts.py | 7 +++++++ 4 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 fal_app.py create mode 100644 prompts.py diff --git a/fal_app.py b/fal_app.py new file mode 100644 index 0000000..bbb1bf2 --- /dev/null +++ b/fal_app.py @@ -0,0 +1,35 @@ +import io +import fal +import torch +from fal.toolkit import File + +from prompts import PROMPTS + + +class InfinifiFalApp(fal.App, keep_alive=300): + machine_type = "GPU-A6000" + requirements = [ + "torch==2.1.0", + "audiocraft==1.3.0", + "torchaudio==2.1.0", + "websockets==11.0.3", + ] + + def setup(self): + import torchaudio + from audiocraft.models.musicgen import MusicGen + + self.model = MusicGen.get_pretrained("facebook/musicgen-large") + self.model.set_generation_params(duration=60) + + @fal.endpoint("/generate") + def run(self): + wav = self.model.generate(PROMPTS) + + serialized = [] + for one_wav in wav: + buf = io.BytesIO() + torch.save(one_wav.cpu(), buf) + serialized.append(buf.getvalue()) + + return serialized diff --git a/generate.py b/generate.py index 6912c9c..146fa03 100644 --- a/generate.py +++ b/generate.py @@ -2,6 +2,8 @@ from audiocraft.models.musicgen import MusicGen from audiocraft.data.audio import audio_write +from prompts import PROMPTS + MODEL_NAME = "facebook/musicgen-large" MUSIC_DURATION_SECONDS = 60 @@ -17,7 +19,7 @@ def generate(offset=0): - wav = model.generate(descriptions) + wav = model.generate(PROMPTS) for idx, one_wav in enumerate(wav): # Will save under {idx}.wav, with loudness normalization at -14 db LUFS. diff --git a/generate_manual.py b/generate_manual.py index 536e02f..fe2b34a 100644 --- a/generate_manual.py +++ b/generate_manual.py @@ -3,6 +3,8 @@ from audiocraft.models.musicgen import MusicGen from audiocraft.data.audio import audio_write +from prompts import PROMPTS + MODEL_NAME = "facebook/musicgen-large" MUSIC_DURATION_SECONDS = 60 @@ -10,18 +12,11 @@ model = MusicGen.get_pretrained(MODEL_NAME) model.set_generation_params(duration=MUSIC_DURATION_SECONDS) -descriptions = [ - "Create a futuristic lo-fi beat that blends modern electronic elements with synthwave influences. Incorporate smooth, atmospheric synths and gentle, relaxing rhythms to evoke a sense of a serene, neon-lit future. Ensure the track is continuous with no background noise or interruptions, maintaining a calm and tranquil atmosphere throughout while adding a touch of retro-futuristic vibes.", - "gentle lo-fi beat with a smooth, mellow piano melody in the background. Ensure there are no background noises or interruptions, maintaining a continuous and seamless flow throughout the track. The beat should be relaxing and tranquil, perfect for a calm and reflective atmosphere.", - "Create an earthy lo-fi beat that evokes a natural, grounded atmosphere. Incorporate organic sounds like soft percussion, rustling leaves, and gentle acoustic instruments. The track should have a warm, soothing rhythm with a continuous flow and no background noise or interruptions, maintaining a calm and reflective ambiance throughout.", - "Create a soothing lo-fi beat featuring gentle, melodic guitar riffs. The guitar should be the focal point, supported by subtle, ambient electronic elements and a smooth, relaxed rhythm. Ensure the track is continuous with no background noise or interruptions, maintaining a warm and mellow atmosphere throughout.", - "Create an ambient lo-fi beat with a tranquil and ethereal atmosphere. Use soft, atmospheric pads, gentle melodies, and minimalistic percussion to evoke a sense of calm and serenity. Ensure the track is continuous with no background noise or interruptions, maintaining a soothing and immersive ambiance throughout.", -] print("model obtained. generating audio...") a = time.time() -wav = model.generate(descriptions) +wav = model.generate(PROMPTS) b = time.time() print(f"audio generated. took {b - a} seconds.") diff --git a/prompts.py b/prompts.py new file mode 100644 index 0000000..6f42bf8 --- /dev/null +++ b/prompts.py @@ -0,0 +1,7 @@ +PROMPTS = [ + "Create a futuristic lo-fi beat that blends modern electronic elements with synthwave influences. Incorporate smooth, atmospheric synths and gentle, relaxing rhythms to evoke a sense of a serene, neon-lit future. Ensure the track is continuous with no background noise or interruptions, maintaining a calm and tranquil atmosphere throughout while adding a touch of retro-futuristic vibes.", + "gentle lo-fi beat with a smooth, mellow piano melody in the background. Ensure there are no background noises or interruptions, maintaining a continuous and seamless flow throughout the track. The beat should be relaxing and tranquil, perfect for a calm and reflective atmosphere.", + "Create an earthy lo-fi beat that evokes a natural, grounded atmosphere. Incorporate organic sounds like soft percussion, rustling leaves, and gentle acoustic instruments. The track should have a warm, soothing rhythm with a continuous flow and no background noise or interruptions, maintaining a calm and reflective ambiance throughout.", + "Create a soothing lo-fi beat featuring gentle, melodic guitar riffs. The guitar should be the focal point, supported by subtle, ambient electronic elements and a smooth, relaxed rhythm. Ensure the track is continuous with no background noise or interruptions, maintaining a warm and mellow atmosphere throughout.", + "Create an ambient lo-fi beat with a tranquil and ethereal atmosphere. Use soft, atmospheric pads, gentle melodies, and minimalistic percussion to evoke a sense of calm and serenity. Ensure the track is continuous with no background noise or interruptions, maintaining a soothing and immersive ambiance throughout.", +]