From 306a5aae518e0e47be96058b6b3d0d1935b9b6c0 Mon Sep 17 00:00:00 2001 From: viola he Date: Tue, 17 Nov 2026 17:42:39 -0500 Subject: [PATCH] Add MIDI input Co-Authored-By: Claude Opus 5.5 --- Cargo.toml | 4 + crates/processing_core/src/error.rs | 6 + crates/processing_midi/src/lib.rs | 292 +++++++++++++++++- crates/processing_pyo3/examples/midi_input.py | 42 +++ crates/processing_pyo3/src/lib.rs | 23 ++ crates/processing_pyo3/src/midi.rs | 135 +++++++- examples/midi_input.rs | 85 +++++ src/prelude.rs | 5 +- 8 files changed, 583 insertions(+), 9 deletions(-) create mode 100644 crates/processing_pyo3/examples/midi_input.py create mode 100644 examples/midi_input.rs diff --git a/Cargo.toml b/Cargo.toml index 5ef3f465..6901d6e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,6 +165,10 @@ path = "examples/pbr.rs" name = "midi" path = "examples/midi.rs" +[[example]] +name = "midi_input" +path = "examples/midi_input.rs" + [[example]] name = "gltf_load" path = "examples/gltf_load.rs" diff --git a/crates/processing_core/src/error.rs b/crates/processing_core/src/error.rs index f7ad743c..232f07d2 100644 --- a/crates/processing_core/src/error.rs +++ b/crates/processing_core/src/error.rs @@ -48,6 +48,12 @@ pub enum ProcessingError { ShaderNotFound, #[error("MIDI port {0} not found")] MidiPortNotFound(usize), + #[error("No MIDI port matching {0:?}; available ports: {1}")] + MidiPortNameNotFound(String, String), + #[error("No MIDI input connected; call midi_connect_input() first")] + MidiInputNotConnected, + #[error("MIDI connection failed: {0}")] + MidiConnectionFailed(String), #[error("CUDA error: {0}")] CudaError(String), #[error("Compute shader not found")] diff --git a/crates/processing_midi/src/lib.rs b/crates/processing_midi/src/lib.rs index c8e9a6ba..390d13b1 100644 --- a/crates/processing_midi/src/lib.rs +++ b/crates/processing_midi/src/lib.rs @@ -1,5 +1,8 @@ use bevy::prelude::*; -use nannou_midi::{MidiOutput, MidiOutputStream, MidiPort, MidiPortDirection}; +use nannou_midi::{ + MidiError, MidiInput, MidiInputStream, MidiOutput, MidiOutputStream, MidiPort, + MidiPortDirection, +}; use processing_core::app_mut; use processing_core::error::{self, Result}; @@ -10,34 +13,179 @@ pub struct MidiPlugin; pub const NOTE_ON: u8 = 0b1001_0000; pub const NOTE_OFF: u8 = 0b1000_0000; +pub const CONTROL_CHANGE: u8 = 0b1011_0000; +pub const PITCH_BEND: u8 = 0b1110_0000; + +/// What a received [`MidiMessage`] means, decoded from its status byte. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MidiMessageKind { + NoteOn, + /// Includes note-on messages with velocity 0, which many devices send instead. + NoteOff, + ControlChange, + PitchBend, + Other, +} + +/// Decode the kind of a MIDI message. +pub fn midi_message_kind(msg: &MidiMessage) -> MidiMessageKind { + if msg.is_note_on() { + MidiMessageKind::NoteOn + } else if msg.is_note_off() { + MidiMessageKind::NoteOff + } else { + match msg.msg[0] & 0xF0 { + CONTROL_CHANGE => MidiMessageKind::ControlChange, + PITCH_BEND => MidiMessageKind::PitchBend, + _ => MidiMessageKind::Other, + } + } +} #[derive(Resource, Default)] struct ActiveOutput(Option); +#[derive(Resource, Default)] +struct ActiveInput(Option); + impl Plugin for MidiPlugin { fn build(&self, app: &mut App) { app.init_resource::(); + app.init_resource::(); app.add_plugins(nannou_midi::MidiPlugin); } } -fn sorted_output_ports(world: &mut World) -> Vec<(Entity, String)> { +fn sorted_ports(world: &mut World, direction: MidiPortDirection) -> Vec<(Entity, String)> { let mut q = world.query::<(Entity, &Name, &MidiPort)>(); let mut ports: Vec<(Entity, String)> = q .iter(world) - .filter(|(_, _, p)| p.direction == MidiPortDirection::Output) + .filter(|(_, _, p)| p.direction == direction) .map(|(e, n, _)| (e, n.as_str().to_string())) .collect(); ports.sort_by(|a, b| a.1.cmp(&b.1)); ports } -pub fn list_ports(world: &mut World) -> Result> { - Ok(sorted_output_ports(world) +fn sorted_output_ports(world: &mut World) -> Vec<(Entity, String)> { + sorted_ports(world, MidiPortDirection::Output) +} + +fn sorted_input_ports(world: &mut World) -> Vec<(Entity, String)> { + sorted_ports(world, MidiPortDirection::Input) +} + +fn numbered(ports: Vec<(Entity, String)>) -> Vec { + ports .into_iter() .enumerate() .map(|(i, (_, name))| format!("{i}: {name}")) - .collect()) + .collect() +} + +pub fn list_ports(world: &mut World) -> Result> { + Ok(numbered(sorted_output_ports(world))) +} + +pub fn list_input_ports(world: &mut World) -> Result> { + Ok(numbered(sorted_input_ports(world))) +} + +/// Which port to connect to: its number in the port list, or part of its name. +#[derive(Clone, Debug)] +pub enum MidiPortSelector { + Index(usize), + Name(String), +} + +/// Find a port by number, or by name: an exact (case-insensitive) match wins, +/// otherwise the first port whose name contains `name`. +fn find_port( + ports: Vec<(Entity, String)>, + selector: &MidiPortSelector, +) -> Result<(Entity, String)> { + match selector { + MidiPortSelector::Index(i) => ports + .get(*i) + .cloned() + .ok_or(error::ProcessingError::MidiPortNotFound(*i)), + MidiPortSelector::Name(name) => { + let needle = name.to_lowercase(); + let exact = ports.iter().find(|(_, n)| n.to_lowercase() == needle); + let partial = || { + ports + .iter() + .find(|(_, n)| n.to_lowercase().contains(&needle)) + }; + exact.or_else(partial).cloned().ok_or_else(|| { + let available = if ports.is_empty() { + "none".to_string() + } else { + ports + .iter() + .map(|(_, n)| n.as_str()) + .collect::>() + .join(", ") + }; + error::ProcessingError::MidiPortNameNotFound(name.clone(), available) + }) + } + } +} + +pub fn connect_input(In(selector): In, world: &mut World) -> Result { + let (port_entity, port_name) = find_port(sorted_input_ports(world), &selector)?; + + disconnect_input(world)?; + + let connection = world + .spawn(( + Name::new("libprocessing input"), + MidiInput { + port: Some(port_entity), + }, + )) + .id(); + world.resource_mut::().0 = Some(connection); + Ok(port_name) +} + +pub fn disconnect_input(world: &mut World) -> Result<()> { + let entity = world.resource_mut::().0.take(); + if let Some(e) = entity + && let Ok(entity_mut) = world.get_entity_mut(e) + { + entity_mut.despawn(); + } + Ok(()) +} + +/// Fails if opening the input connection failed, so the error surfaces at +/// connect time instead of as silence later. +fn check_input_connected(world: &mut World) -> Result<()> { + let entity = world + .resource::() + .0 + .ok_or(error::ProcessingError::MidiInputNotConnected)?; + if let Some(err) = world.get::(entity) { + return Err(error::ProcessingError::MidiConnectionFailed( + err.message.clone(), + )); + } + Ok(()) +} + +/// Drain every message received since the last read. +pub fn read_messages(world: &mut World) -> Result> { + check_input_connected(world)?; + let entity = world + .resource::() + .0 + .ok_or(error::ProcessingError::MidiInputNotConnected)?; + Ok(world + .get_mut::(entity) + .map(|mut stream| stream.read().map(|data| data.message).collect()) + .unwrap_or_default()) } pub fn connect(In(port): In, world: &mut World) -> Result<()> { @@ -126,6 +274,65 @@ pub fn midi_disconnect() -> error::Result<()> { }) } +#[cfg(not(target_arch = "wasm32"))] +pub fn midi_list_input_ports() -> error::Result> { + app_mut(|app| { + let world = app.world_mut(); + world.run_system_cached(list_input_ports).unwrap() + }) +} + +/// Connect to the input port with this number in [`midi_list_input_ports`]. +/// Returns the port's name. +#[cfg(not(target_arch = "wasm32"))] +pub fn midi_connect_input(port: usize) -> error::Result { + connect_input_with(MidiPortSelector::Index(port)) +} + +/// Connect to the first input port whose name matches `name` (case-insensitive, +/// exact match first, then partial), e.g. `"IAC"`. Returns the port's full name. +#[cfg(not(target_arch = "wasm32"))] +pub fn midi_connect_input_by_name(name: &str) -> error::Result { + connect_input_with(MidiPortSelector::Name(name.to_string())) +} + +#[cfg(not(target_arch = "wasm32"))] +fn connect_input_with(selector: MidiPortSelector) -> error::Result { + app_mut(|app| { + let world = app.world_mut(); + let name = world + .run_system_cached_with(connect_input, selector) + .unwrap()?; + // Open the connection now so a failure is reported here, not on first read. + world + .run_system_cached(nannou_midi::native::open_midi_inputs) + .unwrap(); + check_input_connected(world)?; + Ok(name) + }) +} + +#[cfg(not(target_arch = "wasm32"))] +pub fn midi_disconnect_input() -> error::Result<()> { + app_mut(|app| { + let world = app.world_mut(); + world.run_system_cached(disconnect_input).unwrap() + }) +} + +/// Every MIDI message received on the connected input since the last call. +#[cfg(not(target_arch = "wasm32"))] +pub fn midi_read() -> error::Result> { + app_mut(|app| { + let world = app.world_mut(); + // Pull anything still waiting in the channel from midir's callback thread. + world + .run_system_cached(nannou_midi::native::receive_midi_messages) + .unwrap(); + world.run_system_cached(read_messages).unwrap() + }) +} + #[cfg(not(target_arch = "wasm32"))] pub fn midi_note_on(note: u8, velocity: u8) -> error::Result<()> { app_mut(|app| { @@ -180,3 +387,76 @@ pub fn midi_play_notes(note: u8, duration: u64) -> error::Result<()> { Ok(()) }) } + +#[cfg(test)] +mod tests { + use super::*; + + fn kind(bytes: [u8; 3]) -> MidiMessageKind { + midi_message_kind(&MidiMessage::from(bytes)) + } + + #[test] + fn decodes_message_kinds() { + assert_eq!(kind([0x90, 60, 100]), MidiMessageKind::NoteOn); + assert_eq!(kind([0x80, 60, 0]), MidiMessageKind::NoteOff); + // note-on with velocity 0 is a note-off + assert_eq!(kind([0x90, 60, 0]), MidiMessageKind::NoteOff); + assert_eq!(kind([0xB0, 1, 64]), MidiMessageKind::ControlChange); + assert_eq!(kind([0xE0, 0, 64]), MidiMessageKind::PitchBend); + assert_eq!(kind([0xA0, 60, 10]), MidiMessageKind::Other); + } + + fn ports(names: &[&str]) -> Vec<(Entity, String)> { + names + .iter() + .map(|n| (Entity::PLACEHOLDER, n.to_string())) + .collect() + } + + fn found(names: &[&str], selector: MidiPortSelector) -> Option { + find_port(ports(names), &selector).ok().map(|(_, n)| n) + } + + #[test] + fn finds_ports_by_index_and_name() { + let names = ["IAC Driver Bus 1", "IAC Driver Bus 12", "nanoKONTROL2"]; + let by_name = |n: &str| found(&names, MidiPortSelector::Name(n.to_string())); + + assert_eq!( + found(&names, MidiPortSelector::Index(2)).as_deref(), + Some("nanoKONTROL2") + ); + assert_eq!(found(&names, MidiPortSelector::Index(3)), None); + // partial and case-insensitive + assert_eq!(by_name("nano").as_deref(), Some("nanoKONTROL2")); + assert_eq!(by_name("NANOKONTROL").as_deref(), Some("nanoKONTROL2")); + assert_eq!(by_name("missing"), None); + + // an exact match beats an earlier partial one + let names = ["Ableton Synth", "Synth"]; + assert_eq!( + found(&names, MidiPortSelector::Name("synth".to_string())).as_deref(), + Some("Synth") + ); + } + + #[test] + fn name_not_found_lists_available_ports() { + let err = + find_port(ports(&["A", "B"]), &MidiPortSelector::Name("C".to_string())).unwrap_err(); + assert_eq!( + err.to_string(), + r#"No MIDI port matching "C"; available ports: A, B"# + ); + let err = find_port(ports(&[]), &MidiPortSelector::Name("C".to_string())).unwrap_err(); + assert!(err.to_string().ends_with("available ports: none")); + } + + #[test] + fn kind_ignores_channel() { + assert_eq!(kind([0x9F, 60, 100]), MidiMessageKind::NoteOn); + assert_eq!(kind([0xB5, 7, 127]), MidiMessageKind::ControlChange); + assert_eq!(MidiMessage::from([0x9F, 60, 100]).channel(), 15); + } +} diff --git a/crates/processing_pyo3/examples/midi_input.py b/crates/processing_pyo3/examples/midi_input.py new file mode 100644 index 00000000..a258a6b3 --- /dev/null +++ b/crates/processing_pyo3/examples/midi_input.py @@ -0,0 +1,42 @@ +from processing_py import * +import colorsys + +# shows incoming MIDI notes as colored bars (pitch -> x, velocity -> height), +# any control change sets background brightness. + +PORT = 0 + +held = {} # note number = velocity +brightness = 0.05 + +def setup(): + size(640, 400) + no_stroke() + + midi_refresh_ports() + print("MIDI inputs:", midi_list_input_ports()) + print("listening to", midi_connect_input(PORT)) + +def draw(): + global brightness + + # everything received since the last frame + for msg in midi_read(): + if msg.kind == "note_on": + held[msg.note] = msg.velocity + elif msg.kind == "note_off": + held.pop(msg.note, None) + elif msg.kind == "control_change": + brightness = msg.value / 127 * 0.5 + else: + print(msg) + + background(brightness, brightness, brightness + 0.03) + + bar_w = width / 128 + for note, velocity in held.items(): + h = velocity / 127 * height + fill(*colorsys.hsv_to_rgb((note % 12) / 12, 0.7, 0.95)) + rect(note * bar_w, height - h, bar_w, h) + +run() diff --git a/crates/processing_pyo3/src/lib.rs b/crates/processing_pyo3/src/lib.rs index 1d66f5a7..d69685e1 100644 --- a/crates/processing_pyo3/src/lib.rs +++ b/crates/processing_pyo3/src/lib.rs @@ -479,6 +479,8 @@ pub mod processing_py { #[pymodule_export] use super::math::PyVec4; #[pymodule_export] + use super::midi::PyMidiMessage; + #[pymodule_export] use super::monitor::Monitor; #[pymodule_export] use super::particles::Attribute; @@ -2559,6 +2561,27 @@ pub mod processing_py { fn midi_note_off(note: u8) -> PyResult<()> { midi::note_off(note) } + #[pyfunction] + fn midi_list_input_ports() -> PyResult> { + midi::list_input_ports() + } + /// Connect to a MIDI input and return its name. `port` is a number from + /// `midi_list_input_ports()` or part of a port name like `"IAC"`; the + /// default is the first port. + #[pyfunction] + #[pyo3(signature = (port=None))] + fn midi_connect_input(port: Option<&Bound<'_, PyAny>>) -> PyResult { + midi::connect_input(port) + } + #[pyfunction] + fn midi_disconnect_input() -> PyResult<()> { + midi::disconnect_input() + } + /// Every MIDI message received since the last call, oldest first. + #[pyfunction] + fn midi_read() -> PyResult> { + midi::read() + } #[pyfunction] fn key_is_down(key_code: u32) -> PyResult { diff --git a/crates/processing_pyo3/src/midi.rs b/crates/processing_pyo3/src/midi.rs index 494b5fee..d9f855fb 100644 --- a/crates/processing_pyo3/src/midi.rs +++ b/crates/processing_pyo3/src/midi.rs @@ -1,5 +1,8 @@ use processing::prelude::*; -use pyo3::{exceptions::PyRuntimeError, prelude::*}; +use pyo3::{ + exceptions::{PyRuntimeError, PyTypeError}, + prelude::*, +}; pub fn connect(port: usize) -> PyResult<()> { midi_connect(port).map_err(|e| PyRuntimeError::new_err(format!("{e}"))) @@ -22,3 +25,133 @@ pub fn note_on(note: u8, velocity: u8) -> PyResult<()> { pub fn note_off(note: u8) -> PyResult<()> { midi_note_off(note).map_err(|e| PyRuntimeError::new_err(format!("{e}"))) } +pub fn list_input_ports() -> PyResult> { + midi_list_input_ports().map_err(|e| PyRuntimeError::new_err(format!("{e}"))) +} +/// `port` is a number from `midi_list_input_ports()` or part of a port name. +pub fn connect_input(port: Option<&Bound<'_, PyAny>>) -> PyResult { + let result = match port { + None => midi_connect_input(0), + Some(p) => { + if let Ok(index) = p.extract::() { + midi_connect_input(index) + } else if let Ok(name) = p.extract::() { + midi_connect_input_by_name(&name) + } else { + return Err(PyTypeError::new_err( + "port must be a port number or part of a port name", + )); + } + } + }; + result.map_err(|e| PyRuntimeError::new_err(format!("{e}"))) +} +pub fn disconnect_input() -> PyResult<()> { + midi_disconnect_input().map_err(|e| PyRuntimeError::new_err(format!("{e}"))) +} +pub fn read() -> PyResult> { + midi_read() + .map(|msgs| msgs.into_iter().map(PyMidiMessage::from).collect()) + .map_err(|e| PyRuntimeError::new_err(format!("{e}"))) +} + +/// A MIDI message received on the connected input. +/// +/// `kind` is one of `"note_on"`, `"note_off"`, `"control_change"`, +/// `"pitch_bend"` or `"other"`. `channel` is 0-15. +/// +/// The two data bytes have names for the common kinds: `note` and `velocity` +/// for notes, `control` and `value` for control changes. For pitch bend, +/// `bend` combines them into one value from 0 to 16383 (8192 is centered). +#[pyclass(name = "MidiMessage", frozen)] +pub struct PyMidiMessage { + status: u8, + data1: u8, + data2: u8, +} + +impl From for PyMidiMessage { + fn from(msg: MidiMessage) -> Self { + let [status, data1, data2] = msg.msg; + Self { + status, + data1, + data2, + } + } +} + +impl PyMidiMessage { + fn message(&self) -> MidiMessage { + MidiMessage::from([self.status, self.data1, self.data2]) + } +} + +#[pymethods] +impl PyMidiMessage { + #[getter] + fn kind(&self) -> &'static str { + match midi_message_kind(&self.message()) { + MidiMessageKind::NoteOn => "note_on", + MidiMessageKind::NoteOff => "note_off", + MidiMessageKind::ControlChange => "control_change", + MidiMessageKind::PitchBend => "pitch_bend", + MidiMessageKind::Other => "other", + } + } + + #[getter] + fn channel(&self) -> u8 { + self.message().channel() + } + + #[getter] + fn status(&self) -> u8 { + self.status + } + + #[getter] + fn data1(&self) -> u8 { + self.data1 + } + + #[getter] + fn data2(&self) -> u8 { + self.data2 + } + + #[getter] + fn note(&self) -> u8 { + self.data1 + } + + #[getter] + fn velocity(&self) -> u8 { + self.data2 + } + + #[getter] + fn control(&self) -> u8 { + self.data1 + } + + #[getter] + fn value(&self) -> u8 { + self.data2 + } + + #[getter] + fn bend(&self) -> u16 { + (self.data2 as u16) << 7 | self.data1 as u16 + } + + fn __repr__(&self) -> String { + format!( + "MidiMessage(kind={:?}, channel={}, data1={}, data2={})", + self.kind(), + self.channel(), + self.data1, + self.data2 + ) + } +} diff --git a/examples/midi_input.rs b/examples/midi_input.rs new file mode 100644 index 00000000..f44225ef --- /dev/null +++ b/examples/midi_input.rs @@ -0,0 +1,85 @@ +//! Shows incoming MIDI: every held note is a bar (pitch -> x, velocity -> height), +//! and any control change (a knob or slider) sets the background brightness. + +use bevy::color::Color; +use processing::prelude::*; +use processing_glfw::GlfwContext; +use processing_render::render::command::DrawCommand; + +const WIDTH: u32 = 640; +const HEIGHT: u32 = 400; +/// Which MIDI input to use. To pick one by name instead, call +/// `midi_connect_input_by_name("IAC")` below. +const PORT: usize = 0; + +fn main() { + if let Err(e) = sketch() { + eprintln!("Sketch error: {e:?}"); + exit(1).unwrap(); + } + exit(0).unwrap(); +} + +fn sketch() -> error::Result<()> { + let mut glfw_ctx = GlfwContext::new(WIDTH, HEIGHT, false)?; + init(Config::default())?; + + let surface = glfw_ctx.create_surface(WIDTH, HEIGHT, false)?; + let graphics = graphics_create(surface, WIDTH, HEIGHT, TextureFormat::Rgba16Float)?; + + midi_refresh_ports()?; + let ports = midi_list_input_ports()?; + if ports.is_empty() { + eprintln!("no MIDI input ports found — connect a controller or virtual port and retry"); + return Ok(()); + } + println!("MIDI inputs: {ports:?}"); + println!("listening to {}", midi_connect_input(PORT)?); + + // velocity of each held note, indexed by note number + let mut held = [0u8; 128]; + let mut brightness = 0.05; + + while glfw_ctx.poll_events() { + for msg in midi_read()? { + let [_, data1, data2] = msg.msg; + match midi_message_kind(&msg) { + MidiMessageKind::NoteOn => held[data1 as usize] = data2, + MidiMessageKind::NoteOff => held[data1 as usize] = 0, + MidiMessageKind::ControlChange => brightness = data2 as f32 / 127.0 * 0.5, + _ => println!("{msg:?}"), + } + } + + graphics_begin_draw(graphics)?; + graphics_record_command( + graphics, + DrawCommand::BackgroundColor(Color::srgb(brightness, brightness, brightness + 0.03)), + )?; + + let bar_w = WIDTH as f32 / 128.0; + for (note, &velocity) in held.iter().enumerate() { + if velocity == 0 { + continue; + } + let h = velocity as f32 / 127.0 * HEIGHT as f32; + let hue = (note % 12) as f32 / 12.0 * 360.0; + graphics_record_command(graphics, DrawCommand::Fill(Color::hsv(hue, 0.7, 0.95)))?; + graphics_record_command( + graphics, + DrawCommand::Rect { + x: note as f32 * bar_w, + y: HEIGHT as f32 - h, + w: bar_w, + h, + radii: [0.0; 4], + }, + )?; + } + + graphics_end_draw(graphics)?; + } + + midi_disconnect_input()?; + Ok(()) +} diff --git a/src/prelude.rs b/src/prelude.rs index cf6287f8..3275728d 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -6,8 +6,9 @@ pub use processing_core::{config::*, constants, error}; pub use processing_input::*; #[cfg(not(target_arch = "wasm32"))] pub use processing_midi::{ - midi_connect, midi_disconnect, midi_list_ports, midi_note_off, midi_note_on, midi_play_notes, - midi_refresh_ports, + MidiMessage, MidiMessageKind, midi_connect, midi_connect_input, midi_connect_input_by_name, + midi_disconnect, midi_disconnect_input, midi_list_input_ports, midi_list_ports, + midi_message_kind, midi_note_off, midi_note_on, midi_play_notes, midi_read, midi_refresh_ports, }; pub use processing_render::{ render::command::{