From b28b789447838677b8dbde91c1348e08b2ff3441 Mon Sep 17 00:00:00 2001 From: Damien Ronssin Date: Mon, 7 Aug 2023 23:01:54 +0200 Subject: [PATCH 1/3] pitch bend synthesis attempt --- NeuralNote/Source/SynthController.cpp | 44 ++++++++++++--------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/NeuralNote/Source/SynthController.cpp b/NeuralNote/Source/SynthController.cpp index 88f6ac8..d66fc48 100644 --- a/NeuralNote/Source/SynthController.cpp +++ b/NeuralNote/Source/SynthController.cpp @@ -15,18 +15,13 @@ SynthController::SynthController(NeuralNoteAudioProcessor* inProcessor, MPESynth std::vector SynthController::buildMidiEventsVector(const std::vector& inNoteEvents) { - // TODO: Deal with pitch bends - bool INCLUDE_PITCH_BENDS = false; + int pitch_bend_channel = 2; + // Compute size of single event vector size_t num_midi_messages = 0; for (const auto& note_event: inNoteEvents) { - num_midi_messages += 2; - - if (INCLUDE_PITCH_BENDS) { - if (!note_event.bends.empty()) - num_midi_messages += note_event.bends.size(); - } + num_midi_messages += 2 + note_event.bends.size(); } std::vector out(num_midi_messages); @@ -34,23 +29,24 @@ std::vector SynthController::buildMidiEventsVector(const std::vecto size_t i = 0; for (const auto& note_event: inNoteEvents) { - bool include_bends = INCLUDE_PITCH_BENDS && !note_event.bends.empty(); - float first_bend = include_bends ? float(note_event.bends[0]) / 3.0f : 0.0f; - - // TODO: Use different channels if there's a pitch bend - out[i++] = - MidiMessage::noteOn(1, note_event.pitch, (float) note_event.amplitude).withTimeStamp(note_event.startTime); - - if (include_bends) { - for (size_t j = 0; j < note_event.bends.size(); j++) { - out[i++] = - MidiMessage::pitchWheel( - 1, MidiMessage::pitchbendToPitchwheelPos(static_cast(note_event.bends[j]) / 3.0f, 2)) - .withTimeStamp(note_event.startTime + double(j) * 0.011); - } - } + bool has_pitch_bends = !note_event.bends.empty(); - out[i++] = MidiMessage::noteOff(1, note_event.pitch).withTimeStamp(note_event.endTime); + int channel = has_pitch_bends ? pitch_bend_channel : 1; + + if (has_pitch_bends) + pitch_bend_channel = pitch_bend_channel < 16 ? pitch_bend_channel + 1 : 2; + + out[i++] = MidiMessage::noteOn(channel, note_event.pitch, (float) note_event.amplitude) + .withTimeStamp(note_event.startTime); + + for (size_t j = 0; j < note_event.bends.size(); j++) { + out[i++] = + MidiMessage::pitchWheel( + channel, MidiMessage::pitchbendToPitchwheelPos(static_cast(note_event.bends[j]) / 3.0f, 2)) + .withTimeStamp(note_event.startTime + (double) j * 256.0 / BASIC_PITCH_SAMPLE_RATE); + } + + out[i++] = MidiMessage::noteOff(channel, note_event.pitch).withTimeStamp(note_event.endTime); jassert(i <= num_midi_messages); } From 964cdf70060634f49df13f404421bf6ffb26a893 Mon Sep 17 00:00:00 2001 From: Damien Ronssin Date: Sun, 20 Aug 2023 10:56:21 +0200 Subject: [PATCH 2/3] Added pitch bend in setParameters. Better handling of pitch bend channels. --- Lib/Model/BasicPitch.cpp | 7 ++- Lib/Model/BasicPitch.h | 6 ++- NeuralNote/PluginSources/PluginProcessor.cpp | 11 ++-- NeuralNote/Source/SynthController.cpp | 55 ++++++++++++++------ NeuralNote/Source/SynthController.h | 1 + 5 files changed, 57 insertions(+), 23 deletions(-) diff --git a/Lib/Model/BasicPitch.cpp b/Lib/Model/BasicPitch.cpp index 73b815a..f5d937e 100644 --- a/Lib/Model/BasicPitch.cpp +++ b/Lib/Model/BasicPitch.cpp @@ -15,14 +15,17 @@ void BasicPitch::reset() mNumFrames = 0; } -void BasicPitch::setParameters(float inNoteSensibility, float inSplitSensibility, float inMinNoteDurationMs) +void BasicPitch::setParameters(float inNoteSensibility, + float inSplitSensibility, + float inMinNoteDurationMs, + int inPitchBendMode) { mParams.frameThreshold = 1.0f - inNoteSensibility; mParams.onsetThreshold = 1.0f - inSplitSensibility; mParams.minNoteLength = static_cast(std::round(inMinNoteDurationMs * FFT_HOP / BASIC_PITCH_SAMPLE_RATE)); - mParams.pitchBend = MultiPitchBend; + mParams.pitchBend = (PitchBendModes) inPitchBendMode; mParams.melodiaTrick = true; mParams.inferOnsets = true; } diff --git a/Lib/Model/BasicPitch.h b/Lib/Model/BasicPitch.h index f7d51a0..ef62704 100644 --- a/Lib/Model/BasicPitch.h +++ b/Lib/Model/BasicPitch.h @@ -28,8 +28,12 @@ class BasicPitch * @param inNoteSensibility Note sensibility threshold (0.05, 0.95). Higher gives more notes. * @param inSplitSensibility Split sensibility threshold (0.05, 0.95). Higher will split note more, lower will merge close notes with same pitch * @param inMinNoteDurationMs Minimum note duration to keep in ms. + * @param inPitchBendMode Pitch bend mode to use for transcription */ - void setParameters(float inNoteSensibility, float inSplitSensibility, float inMinNoteDurationMs); + void setParameters(float inNoteSensibility, + float inSplitSensibility, + float inMinNoteDurationMs, + int inPitchBendMode); /** * Transcribe the input audio. The note event vector can be obtained after this with getNoteEvents diff --git a/NeuralNote/PluginSources/PluginProcessor.cpp b/NeuralNote/PluginSources/PluginProcessor.cpp index 1357117..99fe174 100644 --- a/NeuralNote/PluginSources/PluginProcessor.cpp +++ b/NeuralNote/PluginSources/PluginProcessor.cpp @@ -138,7 +138,10 @@ const juce::Optional& NeuralNoteAudioProcesso void NeuralNoteAudioProcessor::_runModel() { - mBasicPitch.setParameters(mParameters.noteSensibility, mParameters.splitSensibility, mParameters.minNoteDurationMs); + mBasicPitch.setParameters(mParameters.noteSensibility, + mParameters.splitSensibility, + mParameters.minNoteDurationMs, + mParameters.pitchBendMode); mBasicPitch.transcribeToMIDI( getSourceAudioManager()->getDownsampledSourceAudioForTranscription().getWritePointer(0), @@ -174,8 +177,10 @@ void NeuralNoteAudioProcessor::updateTranscription() jassert(mState == PopulatedAudioAndMidiRegions); if (mState == PopulatedAudioAndMidiRegions) { - mBasicPitch.setParameters( - mParameters.noteSensibility, mParameters.splitSensibility, mParameters.minNoteDurationMs); + mBasicPitch.setParameters(mParameters.noteSensibility, + mParameters.splitSensibility, + mParameters.minNoteDurationMs, + mParameters.pitchBendMode); mBasicPitch.updateMIDI(); updatePostProcessing(); diff --git a/NeuralNote/Source/SynthController.cpp b/NeuralNote/Source/SynthController.cpp index d66fc48..69007df 100644 --- a/NeuralNote/Source/SynthController.cpp +++ b/NeuralNote/Source/SynthController.cpp @@ -11,44 +11,65 @@ SynthController::SynthController(NeuralNoteAudioProcessor* inProcessor, MPESynth { // Set midi buffer size to 200 elements to avoid allocating memory on audio thread. mMidiBuffer.ensureSize(3 * 200); + + mSynth->enableLegacyMode(2); } std::vector SynthController::buildMidiEventsVector(const std::vector& inNoteEvents) { - int pitch_bend_channel = 2; + // For each channel from 2 to 16, indicates after which frame the channel is available for some pitch bend info. + std::array pitch_bend_end_frames; + std::fill(pitch_bend_end_frames.begin(), pitch_bend_end_frames.end(), -1); - // Compute size of single event vector - size_t num_midi_messages = 0; + // Compute max size of single event vector + size_t max_num_midi_messages = 0; for (const auto& note_event: inNoteEvents) { - num_midi_messages += 2 + note_event.bends.size(); + max_num_midi_messages += 2 + note_event.bends.size(); } - std::vector out(num_midi_messages); - - size_t i = 0; + std::vector out; + out.reserve(max_num_midi_messages); for (const auto& note_event: inNoteEvents) { bool has_pitch_bends = !note_event.bends.empty(); - int channel = has_pitch_bends ? pitch_bend_channel : 1; + // Notes with no pitch bend are using channel 1 + int channel = 1; + + if (has_pitch_bends) { + // Find a channel for pitch bend that is available, or if none, use the one that will be available first. + // (this might result in overlapping pitch bends fir different notes on the same channel) + auto pitch_bend_channel_index = + int(std::min_element(pitch_bend_end_frames.begin(), pitch_bend_end_frames.end()) + - pitch_bend_end_frames.begin()); + + channel = pitch_bend_channel_index + 2; - if (has_pitch_bends) - pitch_bend_channel = pitch_bend_channel < 16 ? pitch_bend_channel + 1 : 2; + // Remove all pitch bend events on this channel that have time >= note_event.start_time + if (pitch_bend_end_frames[pitch_bend_channel_index] >= note_event.startFrame) { + auto new_end = std::remove_if(out.begin(), out.end(), [channel, note_event](const MidiMessage& x) { + return x.isPitchWheel() && (x.getChannel() == channel) && (x.getTimeStamp() >= note_event.startTime) + && (x.getTimeStamp() <= note_event.endTime); + }); + + out.erase(new_end, out.end()); + } + + pitch_bend_end_frames[pitch_bend_channel_index] = note_event.endFrame; + } - out[i++] = MidiMessage::noteOn(channel, note_event.pitch, (float) note_event.amplitude) - .withTimeStamp(note_event.startTime); + out.push_back(MidiMessage::noteOn(channel, note_event.pitch, (float) note_event.amplitude) + .withTimeStamp(note_event.startTime)); for (size_t j = 0; j < note_event.bends.size(); j++) { - out[i++] = + out.push_back( MidiMessage::pitchWheel( channel, MidiMessage::pitchbendToPitchwheelPos(static_cast(note_event.bends[j]) / 3.0f, 2)) - .withTimeStamp(note_event.startTime + (double) j * 256.0 / BASIC_PITCH_SAMPLE_RATE); + .withTimeStamp(note_event.startTime + (double) j * 256.0 / BASIC_PITCH_SAMPLE_RATE)); } - - out[i++] = MidiMessage::noteOff(channel, note_event.pitch).withTimeStamp(note_event.endTime); - jassert(i <= num_midi_messages); + out.push_back(MidiMessage::noteOff(channel, note_event.pitch).withTimeStamp(note_event.endTime)); } std::sort(out.begin(), out.end(), [](const MidiMessage& a, const MidiMessage& b) { diff --git a/NeuralNote/Source/SynthController.h b/NeuralNote/Source/SynthController.h index 4fd11c5..c993c50 100644 --- a/NeuralNote/Source/SynthController.h +++ b/NeuralNote/Source/SynthController.h @@ -5,6 +5,7 @@ #ifndef SynthController_h #define SynthController_h +#include #include #include "SynthVoice.h" From 1100ee8644b98a910740e9b2c697467ffe2294fe Mon Sep 17 00:00:00 2001 From: Damien Ronssin Date: Sun, 10 Sep 2023 15:06:20 +0200 Subject: [PATCH 3/3] Added multi pitch bend option. Only remove overlapping pitch bend if not multi pitch bend --- Lib/MidiFile/MidiFileWriter.cpp | 10 +++++++--- Lib/Model/Notes.cpp | 6 +++++- Lib/Model/Notes.h | 1 - NeuralNote/PluginSources/PluginProcessor.cpp | 13 +++++++++++-- NeuralNote/Source/Components/PianoRoll.cpp | 2 +- .../Components/Views/TranscriptionOptionsView.cpp | 2 +- 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/Lib/MidiFile/MidiFileWriter.cpp b/Lib/MidiFile/MidiFileWriter.cpp index aaa7daa..17f7375 100644 --- a/Lib/MidiFile/MidiFileWriter.cpp +++ b/Lib/MidiFile/MidiFileWriter.cpp @@ -41,6 +41,10 @@ bool MidiFileWriter::writeMidiFile(const std::vector& inNoteEvents } } + // Remove overlapping pitch bends + std::vector note_events = inNoteEvents; + Notes::dropOverlappingPitchBends(note_events); + juce::MidiMessageSequence message_sequence; // Set tempo @@ -58,7 +62,7 @@ bool MidiFileWriter::writeMidiFile(const std::vector& inNoteEvents float prev_pitch_bend_semitone = 0.0f; // Add note events - for (auto& note: inNoteEvents) { + for (auto& note: note_events) { auto note_on = juce::MidiMessage::noteOn(1, note.pitch, static_cast(note.amplitude)); note_on.setTimeStamp((note.startTime + start_offset) * tempo / 60.0 * mTicksPerQuarterNote); @@ -71,7 +75,7 @@ bool MidiFileWriter::writeMidiFile(const std::vector& inNoteEvents if (inPitchBendMode == SinglePitchBend) { for (size_t i = 0; i < note.bends.size(); i++) { prev_pitch_bend_semitone = float(note.bends[i]) / 3.0f; - auto pitch_wheel_pos = MidiMessage::pitchbendToPitchwheelPos(prev_pitch_bend_semitone, 4.0f); + auto pitch_wheel_pos = MidiMessage::pitchbendToPitchwheelPos(prev_pitch_bend_semitone, 2.0f); auto pitch_wheel_event = MidiMessage::pitchWheel(1, pitch_wheel_pos); pitch_wheel_event.setTimeStamp((note.startTime + start_offset + i * FFT_HOP / BASIC_PITCH_SAMPLE_RATE) * tempo / 60.0 * mTicksPerQuarterNote); @@ -94,7 +98,7 @@ bool MidiFileWriter::writeMidiFile(const std::vector& inNoteEvents message_sequence.sort(); message_sequence.updateMatchedPairs(); - DBG("Length of note vector: " << inNoteEvents.size()); + DBG("Length of note vector: " << note_events.size()); DBG("NumEvents in message sequence:" << message_sequence.getNumEvents()); // Write midi file diff --git a/Lib/Model/Notes.cpp b/Lib/Model/Notes.cpp index 0a13a75..06f4848 100644 --- a/Lib/Model/Notes.cpp +++ b/Lib/Model/Notes.cpp @@ -17,6 +17,7 @@ std::vector Notes::convert(const std::vector>& ConvertParams inParams) { std::vector events; + // TODO: better preallocation strategy events.reserve(1000); auto n_frames = inNotesPG.size(); @@ -250,7 +251,10 @@ void Notes::_addPitchBends(std::vector& inOutEvents, max = w; } } - event.bends.emplace_back(bend - pb_shift); + + // TODO: use bigger range. + int clamped_bend = jlimit(-6, 6, bend - pb_shift); + event.bends.emplace_back(clamped_bend); } } } \ No newline at end of file diff --git a/Lib/Model/Notes.h b/Lib/Model/Notes.h index a238d1b..0c7274a 100644 --- a/Lib/Model/Notes.h +++ b/Lib/Model/Notes.h @@ -99,7 +99,6 @@ class Notes */ static void mergeOverlappingNotesWithSamePitch(std::vector& inOutEvents) { - sortEvents(inOutEvents); for (int i = 0; i < int(inOutEvents.size()) - 1; i++) { auto& event = inOutEvents[i]; for (auto j = i + 1; j < inOutEvents.size(); j++) { diff --git a/NeuralNote/PluginSources/PluginProcessor.cpp b/NeuralNote/PluginSources/PluginProcessor.cpp index 99fe174..9c6f034 100644 --- a/NeuralNote/PluginSources/PluginProcessor.cpp +++ b/NeuralNote/PluginSources/PluginProcessor.cpp @@ -160,7 +160,11 @@ void NeuralNoteAudioProcessor::_runModel() mPostProcessedNotes = mRhythmOptions.quantize(post_processed_notes); - Notes::dropOverlappingPitchBends(mPostProcessedNotes); + Notes::sortEvents(mPostProcessedNotes); + + if (mParameters.pitchBendMode != MultiPitchBend) + Notes::dropOverlappingPitchBends(mPostProcessedNotes); + Notes::mergeOverlappingNotesWithSamePitch(mPostProcessedNotes); // For the synth @@ -198,6 +202,7 @@ void NeuralNoteAudioProcessor::updatePostProcessing() mParameters.minMidiNote.load(), mParameters.maxMidiNote.load()); + // TODO: preallocate vector of notes and midi vectors auto post_processed_notes = mNoteOptions.process(mBasicPitch.getNoteEvents()); mRhythmOptions.setParameters(RhythmUtils::TimeDivisions(mParameters.rhythmTimeDivision.load()), @@ -205,7 +210,11 @@ void NeuralNoteAudioProcessor::updatePostProcessing() mPostProcessedNotes = mRhythmOptions.quantize(post_processed_notes); - Notes::dropOverlappingPitchBends(mPostProcessedNotes); + Notes::sortEvents(mPostProcessedNotes); + + if (mParameters.pitchBendMode != MultiPitchBend) + Notes::dropOverlappingPitchBends(mPostProcessedNotes); + Notes::mergeOverlappingNotesWithSamePitch(mPostProcessedNotes); // For the synth diff --git a/NeuralNote/Source/Components/PianoRoll.cpp b/NeuralNote/Source/Components/PianoRoll.cpp index 1f09c50..8d2a181 100644 --- a/NeuralNote/Source/Components/PianoRoll.cpp +++ b/NeuralNote/Source/Components/PianoRoll.cpp @@ -71,7 +71,7 @@ void PianoRoll::paint(Graphics& g) g.drawRect(_timeToPixel(start), note_y_start, _timeToPixel(end) - _timeToPixel(start), note_height, 0.5); // Draw pitch bend - if (mProcessor->getCustomParameters()->pitchBendMode == SinglePitchBend) { + if (mProcessor->getCustomParameters()->pitchBendMode != NoPitchBend) { const auto& bends = note_event.bends; if (!note_event.bends.empty()) { diff --git a/NeuralNote/Source/Components/Views/TranscriptionOptionsView.cpp b/NeuralNote/Source/Components/Views/TranscriptionOptionsView.cpp index be5e003..bea8559 100644 --- a/NeuralNote/Source/Components/Views/TranscriptionOptionsView.cpp +++ b/NeuralNote/Source/Components/Views/TranscriptionOptionsView.cpp @@ -42,7 +42,7 @@ TranscriptionOptionsView::TranscriptionOptionsView(NeuralNoteAudioProcessor& pro mPitchBendDropDown = std::make_unique("PITCH BEND"); mPitchBendDropDown->setEditableText(false); mPitchBendDropDown->setJustificationType(juce::Justification::centredRight); - mPitchBendDropDown->addItemList({"No Pitch Bend", "Single Pitch Bend"}, 1); + mPitchBendDropDown->addItemList({"No Pitch Bend", "Single Pitch Bend", "Multi Pitch Bend"}, 1); mPitchBendDropDown->setSelectedItemIndex(mProcessor.getCustomParameters()->pitchBendMode.load()); mPitchBendDropDown->onChange = [this]() { mProcessor.getCustomParameters()->pitchBendMode.store(mPitchBendDropDown->getSelectedItemIndex());