From 28d5505edae76ed91708a895815eecf542c23060 Mon Sep 17 00:00:00 2001 From: Niklas Halle Date: Thu, 3 Dec 2020 17:37:01 +0100 Subject: inital, matleap api based on https://github.com/tomh4/matleap --- .gitignore | 11 +++ Contents.m | 18 +++++ README.md | 119 +++++++++++++++++++++++++++ build.m | 22 +++++ matleap.cpp | 234 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ matleap.h | 96 ++++++++++++++++++++++ matleap.o | Bin 0 -> 35992 bytes matleap_debug.m | 3 + matleap_frame.m | 3 + matleap_version.m | 3 + test.wav | Bin 0 -> 2681644 bytes test_matleap.m | 57 +++++++++++++ theremine | 6 ++ theremine.m | 148 ++++++++++++++++++++++++++++++++++ 14 files changed, 720 insertions(+) create mode 100644 .gitignore create mode 100644 Contents.m create mode 100644 README.md create mode 100644 build.m create mode 100755 matleap.cpp create mode 100755 matleap.h create mode 100644 matleap.o create mode 100644 matleap_debug.m create mode 100644 matleap_frame.m create mode 100644 matleap_version.m create mode 100644 test.wav create mode 100755 test_matleap.m create mode 100644 theremine create mode 100644 theremine.m diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5a2072a --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +LeapSDK +matleap.mexa64 +.vs/ +LeapSDKv4/ +MatLeapC.sln +MatLeapC.vcxproj +MatLeapC.vcxproj.filters +MatLeapC.vcxproj.user +x64/ +*.mexw64 +LeapC.dll diff --git a/Contents.m b/Contents.m new file mode 100644 index 0000000..a8c0533 --- /dev/null +++ b/Contents.m @@ -0,0 +1,18 @@ +% matleap Leap Motion Controller Interface +% ---------------------------------------- +% +% Copyright (C) 2013 Jeff Perry +% +% FUNCTIONS +% --------- +% v=matleap_version Get the version number +% f=matleap_frame Get a frame from the leap motion controller +% +% HISTORY +% ------- +% jsp Wed Nov 20 10:36:45 CST 2013 Created +% +% SEE ALSO +% -------- +% https://www.leapmotion.com/ +% https://developer.leapmotion.com/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..6589cf5 --- /dev/null +++ b/README.md @@ -0,0 +1,119 @@ +# Matleap: MATLAB Interface to the Leap Motion Controller + +This MATLAB mex-file will allow you to get data from a Leap Motion +Controller device. + +In order to build the mex-file, you need to have a C++ compiler +installed, and you need to setup Matlab for building mex-files. + +For more information on setting up Matlab for building mex-files, see +[the Matlab documentation](http://www.mathworks.com/help/matlab/ref/mex.html). + +## Building + +* Install the Leap SDK +* Give the matleap build module access the Leap SDK: + * If you are running **Windows**, copy the LeapSDK directory to the + directory that contains the matleap code. + * If you are running **OS/X** or **Linux**, you can either copy the + directory, or better still, create a symbolic link to the LeapSDK + directory and place it in the same directory as the matleap code, + for example: + +``` + $ ln -s ~/Software/Leap_Developer_Kit/LeapSDK/ ~/matleap/LeapSDK +``` + +* Build the mex file from within MATLAB + +``` + >> build +``` + +## Testing + +``` + >> test_matleap +``` + +## Examples +``` + >> matleap_version + ans = + 0 4 + >> f=matleap_frame + f = + id: 263725 + timestamp: 8.1430e+09 + pointables: [1x5 struct] + >> f.pointables(1).position + ans = + 8.9769 220.3197 -5.8013 + >> f.pointables(1).velocity + ans = + -6.7446 -4.2978 -13.0157 + >> f.pointables(1).direction + ans = + 0.2089 0.3257 -0.9221 + + >> test_matleap + matleap version 0.4 + frame id 309984 + frame timestamp 8767122812 + frame pointables 5 + pointable 1 + id 6 + position -18.529303 279.245270 -9.088086 + velocity -19.887272 -16.154881 -2.646765 + direction 0.098189 0.300899 -0.948588 + pointable 2 + id 7 + ... + pointable 5 + id 10 + position 34.367825 264.370300 41.408348 + velocity -3.097427 13.046532 9.727820 + direction 0.558435 0.059477 -0.827414 + 89 frames + 1.000254 seconds + 88.977400 fps +``` + +## Troubleshooting + +### General + +* Frames are returned with invalid frame data: **This happens when the + motion controller driver is not installed.** + +* Motion controller has slow framerate: **The driver will go into standby + mode if no movement is detected for a long period of time.** + +### OS/X + +* "libLeap.dylib can't load": **The Leap dynamic link library must be made + available to the mex-file executable either by changing your path, + creating a symbolic link to the library, or by copying the library to the + same directory as the mex-file.** For example: + +``` + $ cp '/Applications/Leap Motion.app/Contents/MacOS/libLeap.dylib' path_to_matleap +``` + +### Windows + +* "Invalid MEX-file '...\matleap.mexw64/32': The specified module could not + be found.": **Leap.dll must be made available to the mex-file executeable, + either by adding it to your path or copying it to the same directory as + the mex-file. The leap DLL is located in the LeapSDK\lib\x86 directory on + 32 bit systems and in the LeapSDK\lib\x64 directory on 64 bit systems.** + +### Linux + +* "Invalid MEX-file ... libLeap.so: cannot open shared object file: No such + file or directory": **The leap shared library must be made available to the + mex-file executable.** For example: + +``` + # ln -s /usr/lib/Leap/libLeap.so /usr/lib/ +``` diff --git a/build.m b/build.m new file mode 100644 index 0000000..f5ef2a1 --- /dev/null +++ b/build.m @@ -0,0 +1,22 @@ +% @file build.m +% @brief build the matleap mex module +% @author Jeff Perry +% @version 1.0 +% @date 2013-09-12 + +libdir_switch='-L/usr/lib/Leap/'; + +% create the mex command line +fn='matleap.cpp'; +fprintf('Compiling %s\n',fn); +cmd=['mex',... + ' -I/usr/include ',... + libdir_switch,... + ' -lLeap ',... + fn]; +fprintf('Evaluating "%s"\n',cmd) + +% run mex +eval(cmd) + +fprintf('Done\n') diff --git a/matleap.cpp b/matleap.cpp new file mode 100755 index 0000000..9a3cb25 --- /dev/null +++ b/matleap.cpp @@ -0,0 +1,234 @@ +/// @file matleap.cpp +/// @brief leap motion controller interface +/// @author Jeff Perry +/// @version 1.0 +/// @date 2013-09-12 + +#include "matleap.h" +#include + +// Under Windows, a Leap::Controller must be allocated after the MEX +// startup code has completed. Also, a Leap::Controller must be +// deleted in the function specified by mexAtExit after all global +// destructors are called. If the Leap::Controller is not allocated +// and freed in this way, the MEX function will crash and cause MATLAB +// to hang or close abruptly. Linux and OS/X don't have these +// constraints, and you can just create a global Leap::Controller +// instance. + +// Global instance pointer +matleap::frame_grabber *fg = 0; +int version = 4; // 1: orig, 2: with arm info, 3: with more hand info + +// Exit function +void matleap_exit() { + fg->close_connection(); + delete fg; + fg = 0; +} + +/// @brief process interface arguments +/// +/// @param nlhs matlab mex output interface +/// @param plhs[] matlab mex output interface +/// @param nrhs matlab mex input interface +/// @param prhs[] matlab mex input interface +/// +/// @return command number +int get_command(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { + int command; + // check inputs + switch (nrhs) { + case 1: + command = *mxGetPr(prhs[0]); + break; + case 0: + mexErrMsgTxt("Not enough input arguments"); + default: + mexErrMsgTxt("Too many input arguments"); + } + // check that inputs agree with command + switch (command) { + case -1: { + // set debug command requires 0 outputs + if (nlhs != 0) + mexErrMsgTxt("Wrong number of outputs specified"); + } + break; + case 0: { + // get version command requires 1 outputs + if (nlhs != 0 && nlhs != 1) + mexErrMsgTxt("Wrong number of outputs specified"); + } + break; + case 1: { + // frame grab command only requires one input + if (nrhs > 1) + mexErrMsgTxt("Too many inputs specified"); + // frame grab command requires exactly one output + if (nlhs != 0 && nlhs != 1) + mexErrMsgTxt("Wrong number of outputs specified"); + } + break; + default: + mexErrMsgTxt("An unknown command was specified"); + } + return command; +} + +/// @brief helper function +/// +/// @param v values to fill array with +/// +/// @return created and filled array +mxArray *create_and_fill(const Leap::Vector &v) { + mxArray *p = mxCreateNumericMatrix(1, 3, mxDOUBLE_CLASS, mxREAL); + *(mxGetPr(p) + 0) = v.x; + *(mxGetPr(p) + 1) = v.y; + *(mxGetPr(p) + 2) = v.z; + return p; +} + +/// @brief get a frame from the leap controller +/// +/// @param nlhs matlab mex output interface +/// @param plhs[] matlab mex output interface +void get_frame(int nlhs, mxArray *plhs[]) { + // get the frame + const matleap::frame &f = fg->get_frame(); + // create matlab frame struct + const char *frame_field_names[] = + { + "id", + "timestamp", + "gesture", + "hands", + "version" + }; + int frame_fields = sizeof(frame_field_names) / sizeof(*frame_field_names); + plhs[0] = mxCreateStructMatrix(1, 1, frame_fields, frame_field_names); + // fill the frame struct + mxSetFieldByNumber(plhs[0], 0, 0, mxCreateDoubleScalar(f.id)); + mxSetFieldByNumber(plhs[0], 0, 1, mxCreateDoubleScalar(f.timestamp)); + mxSetFieldByNumber(plhs[0], 0, 2, mxCreateDoubleScalar(f.has_gesture ? 1 : 0)); + + if (f.detectedHands > 0) { + const char *hand_field_names[] = + { + "id", // 0 + "confidence", // 2 + "visible_time", + "pinch_strength", + "grab_strength", + "palm", + "digits" + }; + + int hand_fields = sizeof(hand_field_names) / sizeof(*hand_field_names); + mxArray *p = mxCreateStructMatrix(1, f.detectedHands, hand_fields, hand_field_names); + mxSetFieldByNumber(plhs[0], 0, 3, p); + // 3 because hands is the third (fourth) field name in + // the overall struct we are creating. + + for (int i = 0; i < f.detectedHands; ++i) { + // one by one, get the fields for the hand + mxSetFieldByNumber(p, i, 0, mxCreateDoubleScalar(f.hands[i].id())); + mxSetFieldByNumber(p, i, 1, mxCreateDoubleScalar(f.hands[i].confidence())); + mxSetFieldByNumber(p, i, 2, mxCreateDoubleScalar(f.hands[i].timeVisible())); + mxSetFieldByNumber(p, i, 3, mxCreateDoubleScalar(f.hands[i].pinchStrength())); + mxSetFieldByNumber(p, i, 4, mxCreateDoubleScalar(f.hands[i].grabStrength())); + + // palm + const char *palm_field_names[] = + { + "position", + "stabilized_position", + "velocity", + "normal", + "width", + "direction", + }; + int palm_fields = sizeof(palm_field_names) / sizeof(*palm_field_names); + mxArray *palm = mxCreateStructMatrix(1, 1, palm_fields, palm_field_names); + mxSetFieldByNumber(p, i, 5, palm); + mxSetFieldByNumber(palm, 0, 0, create_and_fill(f.hands[i].palmPosition())); + mxSetFieldByNumber(palm, 0, 1, create_and_fill(f.hands[i].stabilizedPalmPosition())); + mxSetFieldByNumber(palm, 0, 2, create_and_fill(f.hands[i].palmVelocity())); + mxSetFieldByNumber(palm, 0, 3, create_and_fill(f.hands[i].palmNormal())); + mxSetFieldByNumber(palm, 0, 4, mxCreateDoubleScalar(f.hands[i].palmWidth())); + mxSetFieldByNumber(palm, 0, 5, create_and_fill(f.hands[i].direction())); + // get bones for all fingers + const char *digit_field_names[] = + { + "finger_id", // 0 + "is_extended", + "bones", + }; + int digit_fields = sizeof(digit_field_names) / sizeof(*digit_field_names); + Leap::Finger digits[5]; + std::copy(f.hands[i].fingers().begin(), f.hands[i].fingers().end(), std::begin(digits)); + mxArray *d = mxCreateStructMatrix(1, 5, digit_fields, digit_field_names); + mxSetFieldByNumber(p, i, 6, d); + + for (int d_it = 0; d_it < 5; d_it++) { + mxSetFieldByNumber(d, d_it, 0, mxCreateDoubleScalar(digits[d_it].id())); + mxSetFieldByNumber(d, d_it, 1, mxCreateDoubleScalar(digits[d_it].isExtended())); + const char *bone_field_names[] = + { + "prev_joint", // 0 + "next_joint", // 1 + "width",// 2 + "rotation" + }; + int bone_fields = sizeof(bone_field_names) / sizeof(*bone_field_names); + mxArray *bones = mxCreateStructMatrix(1, 4, bone_fields, bone_field_names); + mxSetFieldByNumber(d, d_it, 2, bones); + + auto const boneParts = {Leap::Bone::TYPE_METACARPAL, Leap::Bone::TYPE_PROXIMAL, + Leap::Bone::TYPE_INTERMEDIATE, Leap::Bone::TYPE_DISTAL}; + + Leap::Bone bone; + for (auto bi : boneParts) { + bone = digits[d_it].bone(bi); + + mxSetFieldByNumber(bones, bi, 0, create_and_fill(bone.prevJoint())); // 0 + mxSetFieldByNumber(bones, bi, 1, create_and_fill(bone.nextJoint())); // 1 + mxSetFieldByNumber(bones, bi, 2, mxCreateDoubleScalar(bone.width())); // 2 + mxSetFieldByNumber(bones, bi, 3, create_and_fill(bone.direction())); + + } + } + } // re: for f.hands.count() + } // re: if f.hands.count() > 0 + + mxSetFieldByNumber(plhs[0], 0, 5, mxCreateDoubleScalar(version)); +} + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) { + if (!fg) { + fg = new matleap::frame_grabber; + if (fg == 0) + mexErrMsgTxt("Cannot allocate a frame grabber"); + fg->open_connection(); + mexAtExit(matleap_exit); + } + switch (get_command(nlhs, plhs, nrhs, prhs)) { + // turn on debug + case -1: + fg->set_debug(true); + return; + // get version + case 0: + plhs[0] = mxCreateNumericMatrix(1, 2, mxDOUBLE_CLASS, mxREAL); + *(mxGetPr(plhs[0]) + 0) = MAJOR_REVISION; + *(mxGetPr(plhs[0]) + 1) = MINOR_REVISION; + return; + // get frame + case 1: + get_frame(nlhs, plhs); + return; + default: + // this is a logic error + mexErrMsgTxt("unknown error: please contact developer"); + } +} diff --git a/matleap.h b/matleap.h new file mode 100755 index 0000000..24eae30 --- /dev/null +++ b/matleap.h @@ -0,0 +1,96 @@ +/// @file matleap.h +/// @brief leap motion controller interface +/// @author Jeff Perry +/// @version 1.0 +/// @date 2013-09-12 + +#ifndef MATLEAP_H +#define MATLEAP_H + +#define MAJOR_REVISION 4 +#define MINOR_REVISION 0 + +#include "Leap.h" +#include "mex.h" + +#include + +namespace matleap { + +/// @brief a leap frame + struct frame { + int64_t id; + int64_t timestamp; + uint32_t detectedHands; + Leap::HandList hands; + bool has_gesture; + }; + +/// @brief leap frame grabber interface + class frame_grabber { + private: + bool debug; + Leap::Controller controllerConnection; + frame current_frame; + + public: + /// @brief constructor + frame_grabber() + : debug(false) { + } + + void open_connection() { + mexPrintf("Waiting for connection"); + while (!controllerConnection.isConnected()) { + mexPrintf("."); + usleep(500000); + } + //controllerConnection.enableGesture(Leap::Gesture::TYPE_SWIPE); + controllerConnection.enableGesture(Leap::Gesture::TYPE_CIRCLE); + //controllerConnection.enableGesture(Leap::Gesture::TYPE_KEY_TAP); + //controllerConnection.enableGesture(Leap::Gesture::TYPE_SCREEN_TAP); + mexPrintf(" Connected!\n"); + // TODO: check if needed + //LeapSetPolicyFlags(*controllerConnection, eLeapPolicyFlag_BackgroundFrames, 0); + } + + void close_connection() { + mexPrintf("Good bye."); + } + + /// @brief destructor + ~frame_grabber() { + if (debug) + mexPrintf("Closing matleap frame grabber\n"); + } + + /// @brief debug member access + /// + /// @param flag turn it on/off + void set_debug(bool flag) { + if (flag == debug) + return; + if (flag) + mexPrintf("Setting debug on\n"); + debug = flag; + } + + /// @brief get a frame from the controller + /// + /// @return the frame + frame const &get_frame() { + auto const &frame = controllerConnection.frame(); + current_frame.id = frame.id(); + if (debug) + mexPrintf("Got frame with id %d\n", current_frame.id); + current_frame.timestamp = frame.timestamp(); + current_frame.detectedHands = frame.hands().count(); + current_frame.hands = frame.hands(); + current_frame.has_gesture = !frame.gestures().isEmpty(); + return current_frame; + } + }; + +} // namespace matleap + +#endif diff --git a/matleap.o b/matleap.o new file mode 100644 index 0000000..2e6adcb Binary files /dev/null and b/matleap.o differ diff --git a/matleap_debug.m b/matleap_debug.m new file mode 100644 index 0000000..703687a --- /dev/null +++ b/matleap_debug.m @@ -0,0 +1,3 @@ +function matleap_debug +% MATLEAP_DEBUG Turn on matleap debug information output +matleap(-1); diff --git a/matleap_frame.m b/matleap_frame.m new file mode 100644 index 0000000..53f150d --- /dev/null +++ b/matleap_frame.m @@ -0,0 +1,3 @@ +function f=matleap_frame +% MATLEAP_FRAME Get a frame from the leap motion controller +f=matleap(1); diff --git a/matleap_version.m b/matleap_version.m new file mode 100644 index 0000000..ea5c422 --- /dev/null +++ b/matleap_version.m @@ -0,0 +1,3 @@ +function v=matleap_version +% MATLEAP_VERSION Get matleap version information +v=matleap(0); diff --git a/test.wav b/test.wav new file mode 100644 index 0000000..887bfdf Binary files /dev/null and b/test.wav differ diff --git a/test_matleap.m b/test_matleap.m new file mode 100755 index 0000000..0c6c322 --- /dev/null +++ b/test_matleap.m @@ -0,0 +1,57 @@ +% @file test_matleap.m +% @brief test matleap functionality +% @author Jeff Perry +% @version 1.0 +% @date 2013-09-12 + +function test_matleap + % remove matleap mex-file from memory + % set debug on + %matleap_debug + % show version + [version]=matleap_version; + fprintf('matleap version %d.%d\n',version(1),version(2)); + % pause to let the hardware wake up + sleep(1) + % get some frames + frame_id=-1; + frames=0; + tic + while(toc<10) + % get a frame + f=matleap_frame; + % only count it if it has a different id + if f.id~=frame_id + frame_id=f.id; + print(f) + frames=frames+1; + end + end + s=toc; + % display performance + fprintf('%d frames\n',frames); + fprintf('%f seconds\n',s); + fprintf('%f fps\n',frames/s); +end + +% sleep for t seconds +function sleep(t) + tic; + while (toc 0 + gesture_count = gesture_count + 1; + else + gesture_count = 0; + end + elseif isequal(two_hands, handCount) + gesture_count = 0; + + pos = frame.hands(1).palm.stabilized_position; + signal_one = get_theremin_sound_bit(pos(1), pos(2), signal); + P(count, 1:3) = pos; + count = count + 1; + + pos = frame.hands(2).palm.stabilized_position; + signal_two = get_theremin_sound_bit(pos(1), pos(2), signal); + else + "That should not happen:" + handCount + return; + end + + % play current sound + full_signal = signal_one + signal_two; + complete_signal = [complete_signal full_signal]; + + % enforce frame rate, leaving a bit of room for calculation + java.lang.Thread.sleep((1/(const_frames_per_second - const_calc_offset)) * 1000); +end + +theremin_player = audioplayer(complete_signal, const_Fs); +play(theremin_player); + +% extract values +x = P(:,1); % links (-) rechconst_Ts (+) (LED zu uns) +y = P(:,2); % oben unten +z = P(:,3); % vorne (+) hinten (-) (LED zu uns) + +% plot +figure("Position",[0,0, 1200, 2400]); +t = tiledlayout(4,1); + +nexttile; +plot(x); +ylabel('left right'); + +nexttile; +plot(y); +ylabel('height'); + +nexttile; +plot(z); +ylabel('depth'); + +nexttile; +plot3(z,x,y); +xlabel('depth'); +ylabel('left right'); +zlabel('height'); + +function sound = get_theremin_sound_bit(x, y, generator) + % finding ranges for x and y: + %'x' + %min(x) + %max(x) + %'y' + %min(y) + %max(y) + % --> -300 < x < 300 + % --> 0 < y < 600 + + % therefore + volume = y / 600; + frequency = (x + 300) / 600 * 3000; % https://web.physics.ucsb.edu/~lecturedemonstrations/Composer/Pages/60.17.html # How it works + + sound = generator(frequency, volume); +end \ No newline at end of file -- cgit v1.2.3-54-g00ecf