Synth pitch bends

Lib/MidiFile/MidiFileWriter.cpp

@@ -41,6 +41,10 @@ bool MidiFileWriter::writeMidiFile(const std::vector<Notes::Event>& inNoteEvents
         }
     }
 
+    // Remove overlapping pitch bends
+    std::vector<Notes::Event> note_events = inNoteEvents;
+    Notes::dropOverlappingPitchBends(note_events);
+
     juce::MidiMessageSequence message_sequence;
 
     // Set tempo
@@ -58,7 +62,7 @@ bool MidiFileWriter::writeMidiFile(const std::vector<Notes::Event>& inNoteEvents
     float prev_pitch_bend_semitone = 0.0f;
 
     // Add note events
-    for (auto& note: inNoteEvents) {
+    for (auto& note: note_events) {
         auto note_on = juce::MidiMessage::noteOn(1, note.pitch, static_cast<float>(note.amplitude));
         note_on.setTimeStamp((note.startTime + start_offset) * tempo / 60.0 * mTicksPerQuarterNote);
 
@@ -71,7 +75,7 @@ bool MidiFileWriter::writeMidiFile(const std::vector<Notes::Event>& inNoteEvents
         if (inPitchBendMode == SinglePitchBend) {
             for (size_t i = 0; i < note.bends.size(); i++) {
                 prev_pitch_bend_semitone = float(note.bends[i]) / 3.0f;
-                auto pitch_wheel_pos = MidiMessage::pitchbendToPitchwheelPos(prev_pitch_bend_semitone, 4.0f);
+                auto pitch_wheel_pos = MidiMessage::pitchbendToPitchwheelPos(prev_pitch_bend_semitone, 2.0f);
                 auto pitch_wheel_event = MidiMessage::pitchWheel(1, pitch_wheel_pos);
                 pitch_wheel_event.setTimeStamp((note.startTime + start_offset + i * FFT_HOP / BASIC_PITCH_SAMPLE_RATE)
                                                * tempo / 60.0 * mTicksPerQuarterNote);
@@ -94,7 +98,7 @@ bool MidiFileWriter::writeMidiFile(const std::vector<Notes::Event>& inNoteEvents
     message_sequence.sort();
     message_sequence.updateMatchedPairs();
 
-    DBG("Length of note vector: " << inNoteEvents.size());
+    DBG("Length of note vector: " << note_events.size());
     DBG("NumEvents in message sequence:" << message_sequence.getNumEvents());
 
     // Write midi file

Lib/Model/BasicPitch.cpp

@@ -15,14 +15,17 @@ void BasicPitch::reset()
     mNumFrames = 0;
 }
 
-void BasicPitch::setParameters(float inNoteSensibility, float inSplitSensibility, float inMinNoteDurationMs)
+void BasicPitch::setParameters(float inNoteSensibility,
+                               float inSplitSensibility,
+                               float inMinNoteDurationMs,
+                               int inPitchBendMode)
 {
     mParams.frameThreshold = 1.0f - inNoteSensibility;
     mParams.onsetThreshold = 1.0f - inSplitSensibility;
 
     mParams.minNoteLength = static_cast<int>(std::round(inMinNoteDurationMs * FFT_HOP / BASIC_PITCH_SAMPLE_RATE));
 
-    mParams.pitchBend = MultiPitchBend;
+    mParams.pitchBend = (PitchBendModes) inPitchBendMode;
     mParams.melodiaTrick = true;
     mParams.inferOnsets = true;
 }

Lib/Model/BasicPitch.h

@@ -28,8 +28,12 @@ class BasicPitch
      * @param inNoteSensibility Note sensibility threshold (0.05, 0.95). Higher gives more notes.
      * @param inSplitSensibility Split sensibility threshold (0.05, 0.95). Higher will split note more, lower will merge close notes with same pitch
      * @param inMinNoteDurationMs Minimum note duration to keep in ms.
+     * @param inPitchBendMode Pitch bend mode to use for transcription
      */
-    void setParameters(float inNoteSensibility, float inSplitSensibility, float inMinNoteDurationMs);
+    void setParameters(float inNoteSensibility,
+                       float inSplitSensibility,
+                       float inMinNoteDurationMs,
+                       int inPitchBendMode);
 
     /**
      * Transcribe the input audio. The note event vector can be obtained after this with getNoteEvents

Lib/Model/Notes.cpp

@@ -17,6 +17,7 @@ std::vector<Notes::Event> Notes::convert(const std::vector<std::vector<float>>&
                                          ConvertParams inParams)
 {
     std::vector<Notes::Event> events;
+    // TODO: better preallocation strategy
     events.reserve(1000);
 
     auto n_frames = inNotesPG.size();
@@ -250,7 +251,10 @@ void Notes::_addPitchBends(std::vector<Notes::Event>& inOutEvents,
                     max = w;
                 }
             }
-            event.bends.emplace_back(bend - pb_shift);
+
+            // TODO: use bigger range.
+            int clamped_bend = jlimit(-6, 6, bend - pb_shift);
+            event.bends.emplace_back(clamped_bend);
         }
     }
 }

Lib/Model/Notes.h

@@ -99,7 +99,6 @@ class Notes
      */
     static void mergeOverlappingNotesWithSamePitch(std::vector<Notes::Event>& inOutEvents)
     {
-        sortEvents(inOutEvents);
         for (int i = 0; i < int(inOutEvents.size()) - 1; i++) {
             auto& event = inOutEvents[i];
             for (auto j = i + 1; j < inOutEvents.size(); j++) {

NeuralNote/PluginSources/PluginProcessor.cpp

@@ -138,7 +138,10 @@ const juce::Optional<juce::AudioPlayHead::PositionInfo>& NeuralNoteAudioProcesso
 
 void NeuralNoteAudioProcessor::_runModel()
 {
-    mBasicPitch.setParameters(mParameters.noteSensibility, mParameters.splitSensibility, mParameters.minNoteDurationMs);
+    mBasicPitch.setParameters(mParameters.noteSensibility,
+                              mParameters.splitSensibility,
+                              mParameters.minNoteDurationMs,
+                              mParameters.pitchBendMode);
 
     mBasicPitch.transcribeToMIDI(
         getSourceAudioManager()->getDownsampledSourceAudioForTranscription().getWritePointer(0),
@@ -157,7 +160,11 @@ void NeuralNoteAudioProcessor::_runModel()
 
     mPostProcessedNotes = mRhythmOptions.quantize(post_processed_notes);
 
-    Notes::dropOverlappingPitchBends(mPostProcessedNotes);
+    Notes::sortEvents(mPostProcessedNotes);
+
+    if (mParameters.pitchBendMode != MultiPitchBend)
+        Notes::dropOverlappingPitchBends(mPostProcessedNotes);
+
     Notes::mergeOverlappingNotesWithSamePitch(mPostProcessedNotes);
 
     // For the synth
@@ -174,8 +181,10 @@ void NeuralNoteAudioProcessor::updateTranscription()
     jassert(mState == PopulatedAudioAndMidiRegions);
 
     if (mState == PopulatedAudioAndMidiRegions) {
-        mBasicPitch.setParameters(
-            mParameters.noteSensibility, mParameters.splitSensibility, mParameters.minNoteDurationMs);
+        mBasicPitch.setParameters(mParameters.noteSensibility,
+                                  mParameters.splitSensibility,
+                                  mParameters.minNoteDurationMs,
+                                  mParameters.pitchBendMode);
 
         mBasicPitch.updateMIDI();
         updatePostProcessing();
@@ -193,14 +202,19 @@ void NeuralNoteAudioProcessor::updatePostProcessing()
                                    mParameters.minMidiNote.load(),
                                    mParameters.maxMidiNote.load());
 
+        // TODO: preallocate vector of notes and midi vectors
         auto post_processed_notes = mNoteOptions.process(mBasicPitch.getNoteEvents());
 
         mRhythmOptions.setParameters(RhythmUtils::TimeDivisions(mParameters.rhythmTimeDivision.load()),
                                      mParameters.rhythmQuantizationForce.load());
 
         mPostProcessedNotes = mRhythmOptions.quantize(post_processed_notes);
 
-        Notes::dropOverlappingPitchBends(mPostProcessedNotes);
+        Notes::sortEvents(mPostProcessedNotes);
+
+        if (mParameters.pitchBendMode != MultiPitchBend)
+            Notes::dropOverlappingPitchBends(mPostProcessedNotes);
+
         Notes::mergeOverlappingNotesWithSamePitch(mPostProcessedNotes);
 
         // For the synth

NeuralNote/Source/Components/PianoRoll.cpp

@@ -71,7 +71,7 @@ void PianoRoll::paint(Graphics& g)
             g.drawRect(_timeToPixel(start), note_y_start, _timeToPixel(end) - _timeToPixel(start), note_height, 0.5);
 
             // Draw pitch bend
-            if (mProcessor->getCustomParameters()->pitchBendMode == SinglePitchBend) {
+            if (mProcessor->getCustomParameters()->pitchBendMode != NoPitchBend) {
                 const auto& bends = note_event.bends;
 
                 if (!note_event.bends.empty()) {

NeuralNote/Source/Components/Views/TranscriptionOptionsView.cpp

@@ -42,7 +42,7 @@ TranscriptionOptionsView::TranscriptionOptionsView(NeuralNoteAudioProcessor& pro
     mPitchBendDropDown = std::make_unique<juce::ComboBox>("PITCH BEND");
     mPitchBendDropDown->setEditableText(false);
     mPitchBendDropDown->setJustificationType(juce::Justification::centredRight);
-    mPitchBendDropDown->addItemList({"No Pitch Bend", "Single Pitch Bend"}, 1);
+    mPitchBendDropDown->addItemList({"No Pitch Bend", "Single Pitch Bend", "Multi Pitch Bend"}, 1);
     mPitchBendDropDown->setSelectedItemIndex(mProcessor.getCustomParameters()->pitchBendMode.load());
     mPitchBendDropDown->onChange = [this]() {
         mProcessor.getCustomParameters()->pitchBendMode.store(mPitchBendDropDown->getSelectedItemIndex());

NeuralNote/Source/SynthController.cpp

@@ -11,48 +11,65 @@ SynthController::SynthController(NeuralNoteAudioProcessor* inProcessor, MPESynth
 {
     // Set midi buffer size to 200 elements to avoid allocating memory on audio thread.
     mMidiBuffer.ensureSize(3 * 200);
+
+    mSynth->enableLegacyMode(2);
 }
 
 std::vector<MidiMessage> SynthController::buildMidiEventsVector(const std::vector<Notes::Event>& inNoteEvents)
 {
-    // TODO: Deal with pitch bends
-    bool INCLUDE_PITCH_BENDS = false;
-    // Compute size of single event vector
-    size_t num_midi_messages = 0;
+    // For each channel from 2 to 16, indicates after which frame the channel is available for some pitch bend info.
+    std::array<int, 15> pitch_bend_end_frames;
+    std::fill(pitch_bend_end_frames.begin(), pitch_bend_end_frames.end(), -1);
 
-    for (const auto& note_event: inNoteEvents) {
-        num_midi_messages += 2;
+    // Compute max size of single event vector
+    size_t max_num_midi_messages = 0;
 
-        if (INCLUDE_PITCH_BENDS) {
-            if (!note_event.bends.empty())
-                num_midi_messages += note_event.bends.size();
-        }
+    for (const auto& note_event: inNoteEvents) {
+        max_num_midi_messages += 2 + note_event.bends.size();
     }
 
-    std::vector<MidiMessage> out(num_midi_messages);
-
-    size_t i = 0;
+    std::vector<MidiMessage> out;
+    out.reserve(max_num_midi_messages);
 
     for (const auto& note_event: inNoteEvents) {
-        bool include_bends = INCLUDE_PITCH_BENDS && !note_event.bends.empty();
-        float first_bend = include_bends ? float(note_event.bends[0]) / 3.0f : 0.0f;
-
-        // TODO: Use different channels if there's a pitch bend
-        out[i++] =
-            MidiMessage::noteOn(1, note_event.pitch, (float) note_event.amplitude).withTimeStamp(note_event.startTime);
-
-        if (include_bends) {
-            for (size_t j = 0; j < note_event.bends.size(); j++) {
-                out[i++] =
-                    MidiMessage::pitchWheel(
-                        1, MidiMessage::pitchbendToPitchwheelPos(static_cast<float>(note_event.bends[j]) / 3.0f, 2))
-                        .withTimeStamp(note_event.startTime + double(j) * 0.011);
+        bool has_pitch_bends = !note_event.bends.empty();
+
+        // Notes with no pitch bend are using channel 1
+        int channel = 1;
+
+        if (has_pitch_bends) {
+            // Find a channel for pitch bend that is available, or if none, use the one that will be available first.
+            // (this might result in overlapping pitch bends fir different notes on the same channel)
+            auto pitch_bend_channel_index =
+                int(std::min_element(pitch_bend_end_frames.begin(), pitch_bend_end_frames.end())
+                    - pitch_bend_end_frames.begin());
+
+            channel = pitch_bend_channel_index + 2;
+
+            // Remove all pitch bend events on this channel that have time >= note_event.start_time
+            if (pitch_bend_end_frames[pitch_bend_channel_index] >= note_event.startFrame) {
+                auto new_end = std::remove_if(out.begin(), out.end(), [channel, note_event](const MidiMessage& x) {
+                    return x.isPitchWheel() && (x.getChannel() == channel) && (x.getTimeStamp() >= note_event.startTime)
+                           && (x.getTimeStamp() <= note_event.endTime);
+                });
+
+                out.erase(new_end, out.end());
             }
+
+            pitch_bend_end_frames[pitch_bend_channel_index] = note_event.endFrame;
         }
 
-        out[i++] = MidiMessage::noteOff(1, note_event.pitch).withTimeStamp(note_event.endTime);
+        out.push_back(MidiMessage::noteOn(channel, note_event.pitch, (float) note_event.amplitude)
+                          .withTimeStamp(note_event.startTime));
+
+        for (size_t j = 0; j < note_event.bends.size(); j++) {
+            out.push_back(
+                MidiMessage::pitchWheel(
+                    channel, MidiMessage::pitchbendToPitchwheelPos(static_cast<float>(note_event.bends[j]) / 3.0f, 2))
+                    .withTimeStamp(note_event.startTime + (double) j * 256.0 / BASIC_PITCH_SAMPLE_RATE));
+        }
 
-        jassert(i <= num_midi_messages);
+        out.push_back(MidiMessage::noteOff(channel, note_event.pitch).withTimeStamp(note_event.endTime));
     }
 
     std::sort(out.begin(), out.end(), [](const MidiMessage& a, const MidiMessage& b) {

NeuralNote/Source/SynthController.h

@@ -5,6 +5,7 @@
 #ifndef SynthController_h
 #define SynthController_h
 
+#include <array>
 #include <JuceHeader.h>
 
 #include "SynthVoice.h"