1. create a UIFigure with the desired contents, and add to it (at least) one more dummy control, which has an associated Matlab callback.
2. execute a JS snippet that programmatically interacts with the dummy control, whenever some event-of-interest happens, causing the Matlab callback to fire.
3. query the webWindow, from within the Matlab callback, to retrieve any additional information (“payload”) that the JS passed.

This approach allows, for example, to easily respond to mouse events:

function varargout = jsEventDemo(demoNum)
   % Handle inputs and outputs
   if ~nargin
      demoNum = 4;
   end
   if ~nargout
      varargout = {};
   end
   % Create a simple figure:
   hFig = uifigure('Position',[680,680,330,240],'Resize','off');
   hTA = uitextarea(hFig, 'Value', 'Psst... Come here...!','Editable','off');
   [hWin,idTA] = mlapptools.getWebElements(hTA);
   % Open in browser (DEBUG):
   % mlapptools.waitForFigureReady(hFig); mlapptools.unlockUIFig(hFig); pause(1);
   % web(hWin.URL,'-browser')
   % Prepare the JS command corresponding to the requested demo (1-4)
   switch demoNum
      % Demo #1: Respond to mouse events, inside JS, using "onSomething" bindings:
      case 1
         % Example from: https://dojotoolkit.org/documentation/tutorials/1.10/events/#dom-events
         jsCommand = sprintf(fileread('jsDemo1.js'), idTA.ID_val);
      % Demo #2: Respond to mouse click events, inside JS, using pub/sub:
      case 2
         % Example from: https://dojotoolkit.org/documentation/tutorials/1.10/events/#publish-subscribe
         hTA.Value = 'Click here and see what happens';
         jsCommand = sprintf(fileread('jsDemo2.js'), idTA.ID_val);
      % Demo #3: Trigger Matlab callbacks programmatically from JS by "pressing" a fake button:
      case 3
         hB = uibutton(hFig, 'Visible', 'off', 'Position', [0 0 0 0], ...
                       'ButtonPushedFcn', @fakeButtonCallback);
         [~,idB] = mlapptools.getWebElements(hB);
         jsCommand = sprintf(fileread('jsDemo3.js'), idTA.ID_val, idB.ID_val);
      % Demo 4: Trigger Matlab callbacks and include a "payload" (i.e. eventData) JSON:
      case 4
         hB = uibutton(hFig, 'Visible', 'off', 'Position', [0 0 0 0],...
                      'ButtonPushedFcn', @(varargin)smartFakeCallback(varargin{:}, hWin));
         [~,idB] = mlapptools.getWebElements(hB);
         jsCommand = sprintf(fileread('jsDemo4.js'), idTA.ID_val, idB.ID_val);
   end % switch
   % Execute the JS command
   hWin.executeJS(jsCommand);
end
% Matlab callback function used by Demo #3
function fakeButtonCallback(obj, eventData) %#ok
   disp('Callback activated!');
   pause(2);
end
% Matlab callback function used by Demo #4
function smartFakeCallback(obj, eventData, hWin)
   % Retrieve and decode payload JSON:
   payload = jsondecode(hWin.executeJS('payload'));
   % Print payload summary to the command window:
   disp(['Responding to the fake ' eventData.EventName ...
         ' event with the payload: ' jsonencode(payload) '.']);
   % Update the TextArea
   switch char(payload.action)
      case 'enter',  act_txt = 'entered';
      case 'leave',  act_txt = 'left';
   end
   str = ["Mouse " + act_txt + ' from: '; ...
          "(" + payload.coord(1) + ',' + payload.coord(2) + ')'];
   obj.Parent.Children(2).Value = str;
end

Several thoughts:

• The attached .js files will not work by themselves, rather, they require sprintf to replace the %s with valid widget IDs. Of course, these could be made into proper JS functions.
• Instead of loading the JS files using fileread, we could place the JS code directly in the jsCommand variable, as a Matlab string (char array)
• I tried getting it to work with a textarea control, so that we would get the payload right in the callback’s eventData object in Matlab, Unfortunately, I couldn’t get it to fire programmatically (solutions like this didn’t work). So instead, I store the payload as JSON, and retrieve it with jsondecode(hWin.executeJS('payload')) in the smartFakeCallback function.

JavaScript files

1. jsDemo1.js (direct download link):

   require(["dojo/on", "dojo/dom", "dojo/dom-style", "dojo/mouse"],
   	function(on, dom, domStyle, mouse) {
   		var myDiv = dom.byId("%s");
   		on(myDiv, mouse.enter, function(evt){
   			domStyle.set(myDiv, "backgroundColor", "red");
   		});
   		on(myDiv, mouse.leave, function(evt){
   			domStyle.set(myDiv, "backgroundColor", "");
   		});
   	});
   

2. jsDemo2.js (direct download link):

   require(["dojo/on", "dojo/topic", "dojo/dom"],
   	function(on, topic, dom) {
   		var myDiv = dom.byId("%s");
   		on(myDiv, "click", function() {
   			topic.publish("alertUser", "Your click was converted into an alert!");
   		});
   		topic.subscribe("alertUser", function(text){
   			alert(text);
   		});
   	});
   

3. jsDemo3.js (direct download link):

   require(["dojo/on", "dojo/dom", "dojo/dom-style", "dojo/mouse"],
   	function(on, dom, domStyle, mouse) {
   		var myDiv = dom.byId("%s");
   		var fakeButton = dom.byId("%s");
   		on(myDiv, mouse.enter, function(evt){
   			fakeButton.click();
   		});
   	});
   

4. jsDemo4.js (direct download link):

   var payload = [];
   require(["dojo/on", "dojo/dom", "dojo/topic", "dojo/mouse"],
   	function(on, dom, topic, mouse) {
   		var myDiv = dom.byId("%s");
   		var fakeButton = dom.byId("%s");
   		topic.subscribe("sendToMatlab", function(data){
   			payload = data;
   			fakeButton.click();
   		});
   		on(myDiv, mouse.enter, function(evt){
   			data = {action: "enter",
   				coord: [evt.clientX, evt.clientY]};
   			topic.publish("sendToMatlab", data);
   		});
   		on(myDiv, mouse.leave, function(evt){
   			data = {action: "leave",
   				coord: [evt.clientX, evt.clientY]};
   			topic.publish("sendToMatlab", data);
   		});
   	});
   

Conclusions

As you can see, this opens some interesting possibilities, and I encourage you to experiment with them yourself! This feature will likely be added to the mlapptools toolbox as soon as an intuitive API is conceived.
If you have any comments or questions about the code above, or just want to tell me how you harnessed this mechanism to upgrade your uifigure (I would love to hear about it!), feel free to leave a message below the gist on which this post is based (this way I get notifications!).

The post Matlab callbacks for uifigure JavaScript events appeared first on Undocumented Matlab.

Motivation

As a retinal physiologist, I spend a lot of time in Matlab creating GUIs to visualize and analyze electrophysiological data. The data often requires a lot of processing and quality control checks before it can be used for interpretation and publication. Consequently, I end up with many control elements taking up precious space on my GUI.
In Java-based (legacy/GUIDE) figures, this wasn’t a huge problem because, depending on what GUI components I needed, I could use a pure Matlab approach (a child panel within a parent panel, with a couple of control sliders moving the child panel around), or a number of Java approaches (which are always more fun; Yair described such an approach last week).
Unfortunately, the web-based (App-Designer) figure framework doesn’t support Java, and the pure/documented Matlab approach just doesn’t look good or function very well:

AppDesigner uislider is not a good scrollbar, no matter what we do to it!

A simple command-window example

Since we are building on the series of uifigure customizations, I expect you have already read the preceding related Undocumented Matlab posts:

1. Customizing uifigures part 1
2. Customizing uifigures part 2
3. Customizing uifigures part 3

Also, I highly recommend cloning (or at least downloading) the mlapptools toolbox repo on Github (thanks Iliya Romm et al.). We will use it to simplify life today.

Using the mlapptools toolbox, we need just a few lines of code to set up a scrollable panel. The important thing is knowing how big the panel needs to be to hold all of our control objects. Once we know this, we simply set the panel’s Position property accordingly. Then we can use simple CSS to display scrollbars and define the viewing dimensions.
For example, if we need 260px in width by 720px in height to hold our control widgets, but only have 200px height available, we can solve this problem in 3 steps:

1. Set the uipanel‘s Dimension property to be 260px wide by 720px tall.
2. Set the viewing height using mlapptools.setStyle(scrollPane,'height','200px') for the panel’s CSS height style attribute.
3. Display the vertical scrollbar by calling mlapptools.setStyle(scrollPane,'overflow-y','scroll') for the panel’s CSS overflow-y='scroll' style attribute.

% Create the figure
fig = uifigure('Name', 'Scroll Panel Test');
% Place a uipanel on the figure, color it for fun
scrollPane = uipanel(fig, 'BackgroundColor', [0.5,0.4,1]);
% Define the space requirements for the controls
totalWidth  = 260; %px
totalHeight = 720; %px
viewHeight  = 200; %px
% STEP 1: set the uipanel's Position property to the required dimensions
scrollPane.Position(3) = totalWidth;  % WIDTH
scrollPane.Position(4) = totalHeight; % HEIGHT
% STEP 2: set the viewing height
mlapptools.setStyle(scrollPane, 'height', sprintf('%dpx', viewHeight));
% STEP 3: display the vertical scrollbar
mlapptools.setStyle(scrollPane, 'overflow-y', 'scroll');
% Add control elements into the uipanel
...

Example scrollable uipanel in a Matlab uifigure

Styling the scrollbars

The mlapptools toolbox utilizes dojo.js to query the DOM and set styles, which is essentially setting inline styles on the DOM element, i.e. <div ... style="color: red;background:#FEFEFE;..."> ... </div>. This has the added benefit of overriding any CSS classes Matlab is imposing on the DOM (see CSS precedence). We can inject our own classes into the page’s <head> tag and then use dojo.setClass() to apply our classes to specific GUI elements. For example, we can style our bland scrollbars by using CSS to define a custom scrollpane style class:

/* CSS To stylize the scrollbar */
.scrollpane::-webkit-scrollbar {
  width: 20px;
}
/* Track */
.scrollpane::-webkit-scrollbar-track {
  background-color: white;
  box-shadow: inset 0 0 5px white;
  border-radius: 10px;
}
/* Handle */
.scrollpane::-webkit-scrollbar-thumb {
  background: red;
  border-radius: 10px;
}
/* Handle on hover */
.scrollpane::-webkit-scrollbar-thumb:hover {
  background: #b30000;
}

To get the CSS into the document header, we need to first convert (“stringify”) it in Matlab:

% CSS styles 'stringified' for MATLAB
%  note that the whole style tag is wrapped in single quotes, that is required!
%  i.e. '' which prints to the command window as:
%   ''''
cssText = [...
  '''''' ...
  ];

As explained in Customizing uifigures part 1, we can execute JavaScript (JS) commands through the webWindow object. To simplify it, we use the method from Customizing uifigures part 2 to obtain the webWindow object for our GUI window and and use the executeJS() method to add our HTML into the document’s <head> tag. It is worth checking out the properties and methods of the JS document object.

% get the webWindow object
webWindow = mlapptools.getWebWindow(fig);
% inject the CSS
webWindow.executeJS(['document.head.innerHTML += ',cssText]);

Now the tricky part is that we have to assign our new CSS scrollpane class to our uipanel. We need 2 things: the webWindow object and the data-tag (our panel’s unique ID) attribute.

% get the uipanel data-tag attr.
[webWindow, panelID] = mlapptools.getWebElements(scrollPane);
%make a dojo.js statement (use addClass method)
setClassString = sprintf(...
  'dojo.addClass(dojo.query("[%s = ''%s'']")[0], "%s")',...
  panelID.ID_attr, panelID.ID_val, 'scrollpane');
% add class to DOM element
webWindow.executeJS(setClassString);

The results of applying our scrollpane CSS style on our scroll-pane’s scrollbars

Styling the uipanel

Each uipanel appears to be composed of 4 <div> HTML elements: a wrapper, internal container for the panel title, a separator, and the panel’s body (contents). We first use mlapptools.getWebElements() to get the data-tag ID for the wrapper node. We can then apply styles to the wrapper, or any child node, with CSS and JS.
For example, we can use CSS3 patterns (such as one in this CSS3 gallery) to liven up our panel. We can also use CSS to define a block element that will replace the title container with some static text. The CSS below would be appended to the cssText string we made for styling the scrollbars above:

/* DECORATE THE BACKGROUND */
/* Stars modified from: http://lea.verou.me/css3patterns/#stars */
.scrollpane {
  overflow: auto;
  background:
  linear-gradient(324deg, #232927 4%,   transparent 4%)   -70px 43px,
  linear-gradient( 36deg, #232927 4%,   transparent 4%)    30px 43px,
  linear-gradient( 72deg, #e3d7bf 8.5%, transparent 8.5%)  30px 43px,
  linear-gradient(288deg, #e3d7bf 8.5%, transparent 8.5%) -70px 43px,
  linear-gradient(216deg, #e3d7bf 7.5%, transparent 7.5%) -70px 23px,
  linear-gradient(144deg, #e3d7bf 7.5%, transparent 7.5%)  30px 23px,
  linear-gradient(324deg, #232927 4%,   transparent 4%)   -20px 93px,
  linear-gradient( 36deg, #232927 4%,   transparent 4%)    80px 93px,
  linear-gradient( 72deg, #e3d7bf 8.5%, transparent 8.5%)  80px 93px,
  linear-gradient(288deg, #e3d7bf 8.5%, transparent 8.5%) -20px 93px,
  linear-gradient(216deg, #e3d7bf 7.5%, transparent 7.5%) -20px 73px,
  linear-gradient(144deg, #e3d7bf 7.5%, transparent 7.5%)  80px 73px !important;
  background-color: #232977 !important;
  background-size: 100px 100px !important;
}
/* STYLES TO CENTER A TEXT BLOCK ON A WHITE SEMI-TRANSPARENT BACKGROUND BLOCK */
/* White block div */
.blockdiv {
  color: black;
  font: 25px Arial, sans-serif !important;
  background: rgba(255,255,255,0.75);
  width: 100%;
  height: 30px;
}
/* Text container centered in .blockdiv */
.textdiv {
  position: relative;
  float: left;
  top: 50%;
  left: 50%;
  transform: translate(-50%, -50%);
}

To insert a static text element and its container, we can create a small JS routine that creates parent and child nodes that replace the panel’s title container:

% Make a nodeID string to make the JS call more generic
nodeID = sprintf('''[%s="%s"]''', panelID.ID_attr, panelID.ID_val);
% JS that creates a div within a div, each with their own classes
% The inner div gets the text and is centered within the outer div
% These elements are added before the node MATLAB will use for any controls
% added to scrollPane
js = sprintf( ...
  [ ...
    'var d = document.createElement("div");', ...
    'var t = document.createElement("div");', ...
    'd.classList.add("blockdiv");',...
    't.classList.add("textdiv");', ...
    't.innerHTML= "Some Static Text";', ...
    'd.appendChild(t);', ...
    'document.querySelectorAll(%s)[0]',...
    '.replaceChild(d,document.querySelectorAll(%s)[0].firstChild);' ...
  ], ...
  nodeID, nodeID ...
);
% execute the JS
webWindow.executeJS(js);

Panel background and static elements

Quick recap

So let’s restate the process:

1. Create a uipanel with the Position property set accordingly large enough for your control elements.
2. Use mlapptools.setStyle() to set the overflow style attribute as desired.
3. Use mlapptools.setStyle() to set the width and/or height style attributes to the viewing size (this is how big the viewing area of the panel needs to be in order to fit nicely in your app).
4. Add your control elements with the scrolling uipanel as the parent.
5. If you want some special styles, create a stylesheet and inject it into the <head> and be sure to add the class to your panel’s HTML classList.

The order of items 2-4 are not really important. You just need to ensure that the panel is large enough (via its Position property) to include all your elements/controls.
I really hope that one day soon MathWorks will add CSS and JS hooks to uifigure GUI components (perhaps as settable CSS/JS properties that accept strings?), so that Matlab users could attach their own CSS and JS customizations directly within AppDesigner, without having to go through such undocumented hoops as I’ve shown here. In Loren Shure’s latest blog post, Matlab product manager Dave Garisson indicated that this is indeed planned for a near-future Matlab release (at least for JS, but hopefully also for CSS):

“we are also investigating ways to provide a documented solution for integrating 3rd party JavaScript components in MATLAB apps.”

A complete working example

I created a complete working example in Matlab’s App Designer while figuring this whole thing out. The code (CWE.m) can be downloaded here, and then run directly from Matlab’s command window. Alternatively, the corresponding App Designer file (CWE.mlapp) can be downloaded here. You are welcome to use/adapt the code in your own project. Just to be clear, I love wild colors and crazy themes, but I don’t recommend going this overboard for a real project.

Running app demo

The post Customizing web-GUI uipanel appeared first on Undocumented Matlab.

Overview

The default Keyboard shortcuts listed below for the features can be customized. Most variables can be customized as well (I will point out which variables are not [yet] customizable).
Most GUIs have a search field. Within this search field you can move the list or the tree up and down using arrow keys, or hit <escape> to return to editor. These search fields allow you to enter regular expressions to limit results shown in list or tree. Also, most GUIs are dockable.
The Editor-Plugin utility is open-source. It is available on GitHub and also mirrored on the Matlab File Exchange. A detailed setup guide is provided on the utility’s wiki section in GitHub.
If you discover any problem or have any suggestion for improvement, please visit the utility’s Issues section on GitHub, where all open/closed issues can be tracked and discussed.
A brief overview of some of the features is presented below. For a detailed explanation of these and other features (which are not listed below), please review the Features section of the utility’s wiki (you guessed it: on GitHub…).

Editing

• Delete / duplicate lines – CTRL + SHIFT + Y or D allows you to delete or duplicate current line.
  
• Move lines up or down – CTRL + ALT + UP or DOWN allows you to move selected lines up or down.
  
• Live (auto-replace) templates – Live Templates are editor commands you can design to insert predefined code. Here’s an example for the command %this% (delivered within the package). When you type a command into the editor the string will get replaced by the predefined text. This predefined text may include variables depending on what you want to achieve. %this% was designed to insert the fully qualified name of the current class you’re in (or function, or script).
  
• Clipboard Stack – CTRL + SHIFT + V opens the Clipboard Stack, where the last 10 copied/cut text are stored and can be directly inserted into the current editor position. The Clipboard Stack only stores text copied from editor.

Navigation

• Auto switch current folder and detail viewer – Switching editor tabs will update the Desktop’s current folder and detail viewer. This behavior can be directly changed in the detail viewer bar.
  Use the new icons in the Desktop’s Current Folder panel: to toggle detail viewer and to toggle following the current editor file.
  
• File Structure – I personally use this one the most: CTRL + F12 will show a GUI that let you search methods and properties including inherited ones.
  
• Navigation History – Every editor file that is opened is stored in the navigation history. Up to 50 editor file paths are stored. If you have a 5-Button Mouse, you can navigate through previous location (backward and forward).
• Recently Closed – CTRL + SHIFT + T will show a GUI that allows you to reopen closed editors this or last session.
• Bookmarks – CTRL + SHIFT + F2 will show a GUI that allows you to name, search and delete bookmarks. Matlab has this nice feature to delete bookmarks after closing the editor. This feature will store all bookmarks. If an editor or Matlab is closed and opened later. All Bookmarks will be restored.
  This feature has some issues though. The most obvious first: if the source file has been changed outside of Matlab, the bookmark does not get updated, and may point to the wrong line, or gets deleted. Also on some Systems the default shortcut does not work. But there are workarounds.

Other useful features

• Dockable Windows – As mentioned before, most GUIs are dockable.
  
• Preferences – Preferences panel integrated in Matlab’s main Preferences window.
  
• Execute current line – SHIFT + F9 allows you to execute the current line. This is equivalent to selecting the entire line, then clicking F9, and is similar in concept to the editor’s built-in ability to execute the current cell-block.
  
• VarDiff – Select two variables in the workspace and compare them using Matlab’s internal comparison feature (adding a custom hook to the Workspace context-menu was discussed by Yair back in 2010).
  
• KeyPressListener – Allows you to create custom keyboard shortcuts for your own functions. A similar functionality was discussed by Yair back in 2009.

As noted above, a detailed explanation of these and other features is provided in the Features section of the utility’s wiki. If you discover any problem or have any suggestion for improvement, please visit the utility’s Issues section on GitHub, where issues can be tracked and discussed.

The post Spicing up the Matlab Editor appeared first on Undocumented Matlab.

Don’t fix it, if it ain’t broke…

In Applied Materials Israel (PDC) we use Matlab code for both algorithm development and deployment (production). As part of the dev-ops build system, which builds our product software versions, we build Matlab artifacts (binaries) from the Matlab source code.
A typical software version has several hundreds Matlab artifacts that are automatically rebuilt on a daily basis, and we have many such versions – totaling many thousands of compilations each day.
This process takes a long time, so we were looking for a way to make it more efficient.
The idea that we chose to implement sounds simple – take a single binary module in any software version (Ex. foo.exe – Matlab-compiled exe) and check it: if the source code for this module has not changed since the last compilation then simply don’t compile it, just copy it from previous software version repository. Since most of our code doesn’t change daily (some of it hasn’t changed in years), we can skip the compilation time of most binaries and just copy them from some repository of previously compiled binaries.

In a broader look, avoiding lengthy compilations cycles by not compiling unchanged code is a common programming practice, implemented by all modern compilers. For example, the ‘make’ utility uses a ‘makefile’ to check the time stamps of all dependencies of every object file in order to decide which object requires recompilation. In reality, this is not always the best solution as time stamps may be incorrect, but it works well in the vast majority of cases.
Coming back to Matlab, now comes the hard part – how could our build system know that nothing has changed in module X and that something has changed in module Y? How does it even know which source files it needs to ensure didn’t change?
The credit for the idea goes to my manager, Lior Cohen, as follows: You can actually check the dependency of a given binary after compilation. The basis of the solution is that a Matlab executable is in fact a compressed (zip) file. The idea is then to:

1. Compile the binary once
2. Unzip the binary and “see” all your dependencies (source files are encrypted and resources are not, but we only need the list of file names – not their content).
3. Now build a list of all your dependency files and compute the CRC value of each from the source control. Save it for the next time you are required to compile this module.
4. In the next compilation cycle, find this dependency list, review it, dependency source file at a time and make sure CRC of the dependency hasn’t changed since last time.
5. If no dependency CRC has changed, then copy the binary from the repository of previous software version, without compiling.
6. Otherwise, recompile the binary and rebuild the CRC list of all dependencies again, in preparation for the next compilation cycle.

That’s it! That simple? Well… not really – the reality is a bit more complex since there are many other dependencies that need to be checked. Some of them are:

1. Did the requested Matlab version of the binary change since the last compilation?
2. Did the compilation instructions themselves (we have a sort of ‘makefile’) change?

Basically, I implemented a policy that if anything changed, or if the dependency check itself failed, then we don’t take any chances and just compile this binary. Keeping in mind that this dependencies check and file copying is much faster than a Matlab compilation, we save a lot of actual compilation time using this method.
Bottom line: Given a software version containing hundreds of compilation instructions to execute and assuming not much has changed in the version (which is often the case), we skip over 90% of compilations altogether and only rebuild what really changed. The result is a version build that takes about half an hour, instead of many hours. Moreover, since the compilation process is working significantly less, we get fewer failures, fewer stuck or crashed mcc processes, and [not less importantly] less maintenance required by me.
Note that in our implementation we rely on the undocumented fact that Matlab binaries are in fact compressed zip archives. If and when a future Matlab release will change the implementation such that the binaries will no longer be zip archives, another way will need to be devised in order to ensure the consistency of the target executable with its dependent source files.

Don’t kill it, if it ain’t bad…

I want to share a very weird issue I investigated over a year ago when using Matlab compiled exe. It started with a user showed me a Matlab compiled exe that didn’t run – I’m not talking about a regular Matlab exception: the process was crashing with an MS Windows popup window popping, stating something very obscure.
It was a very weird behavior that I couldn’t explain – the compiler seemed to work well but the compiled executable process kept crashing. Compiling completely different code showed the same behavior.
This issue has to do with the system compiler configuration that is being used. As you might know, when installing the Matlab compiler, before the first compilation is ever made, the user has to state the C compiler that the Matlab compiler should use in its compilation process. This is done by command ‘mbuild –setup’. This command asks the users to choose the C compiler and saves the configuration (batch file back then, xml in the newer versions of Matlab) in the user’s prefdir folder. At the time we were using Microsoft Visual C++ compiler 9.0 SP1.
The breakthrough in the investigation came when I ran mcc command with –verbose flag, which outputs much more compilation info than I would typically ever want… I discovered that although the target executable file had been created, a post compilation step failed to execute, while issuing a very cryptic error message:

mt.exe : general error c101008d: Failed to write the updated manifest to the resource of file “…”. Access is denied.

set POSTLINK_CMDS2=mt.exe -outputresource: %MBUILD_OUTPUT_FILE_NAME%;%MANIFEST_RESOURCE% -manifest "%MANIFEST_FILE_NAME%"

This line of code takes an XML manifest file and inserts it into the generated binary file (additional details).
If you open a valid R2010a (and probably other old versions as well) Matlab-generated exe in a text editor you can actually see a small XML code embedded in it, while in a non-functioning exe I could not see this XML code.
So why would this command fail?
It turned out, as funny as it sounds, to be an antivirus issue – our IT department updated its antivirus policies and this ‘post link’ command suddenly became an illegal operation. Once our IT eased the policy, this command worked well again and the compiled executables stopped crashing, to our great joy.

The post Matlab compilation quirks – take 2 appeared first on Undocumented Matlab.

Data serialization quirks

Dimitri Shvorob may not appear at first glance to be a prolific contributor on Matlab Central, but from the little he has posted over the years I regard him to be a Matlab power-user. So when Dimitri reports something, I take it seriously. Such was the case several months ago, when he contacted me regarding very odd behavior that he saw in his code: the for loop worked well, but the parfor version returned different (incorrect) results. Eventually, Dimitry traced the problem to something originally reported by Dan Austin on his Fluffy Nuke It blog.
The core issue is that if we have a class object that is used within a for loop, Matlab can access the object directly in memory. But with a parfor loop, the object needs to be serialized in order to be sent over to the parallel workers, and deserialized within each worker. If this serialization/deserialization process involves internal class methods, the workers might see a different version of the class object than the one seen in the serial for loop. This could happen, for example, if the serialization/deserialization method croaks on an error, or depends on some dynamic (or random) conditions to create data.
In other words, when we use data objects in a parfor loop, the data object is not necessarily sent “as-is”: additional processing may be involved under the hood that modify the data in a way that may be invisible to the user (or the loop code), resulting in different processing results of the parallel (parfor) vs. serial (for) loops.
For additional aspects of Matlab serialization/deserialization, see my article from 2 years ago (and its interesting feedback comments).

Data precision quirks

The following section was contributed by guest blogger Lior Perlmuter-Shoshany, head algorithmician at a private equity fund.
In my work, I had to work with matrixes in the order of 10⁹ cells. To reduce the memory footprint (and hopefully also improve performance), I decided to work with data of type single instead of Matlab’s default double. Furthermore, in order to speed up the calculation I use parfor rather than for in the main calculation. In the end of the run I am running a mini for-loop to see the best results.
What I discovered to my surprise is that the results from the parfor and for loop variants is not the same!

The following simplified code snippet illustrate the problem by calculating a simple standard-deviation (std) over the same data, in both single– and double-precision. Note that the loops are ran with only a single iteration, to illustrate the fact that the problem is with the parallelization mechanism (probably the serialization/deserialization parts once again), not with the distribution of iterations among the workers.

clear
rng('shuffle','twister');
% Prepare the data in both double and single precision
arr_double = rand(1,100000000);
arr_single = single(arr_double);
% No loop - direct computation
std_single0 = std(arr_single);
std_double0 = std(arr_double);
% Loop #1 - serial for loop
std_single = 0;
std_double = 0;
for i=1
    std_single(i) = std(arr_single);
    std_double(i) = std(arr_double);
end
% Loop #2 - parallel parfor loop
par_std_single = 0;
par_std_double = 0;
parfor i=1
    par_std_single(i) = std(arr_single);
    par_std_double(i) = std(arr_double);
end
% Compare results of for loop vs. non-looped computation
isForSingleOk = isequal(std_single, std_single0)
isForDoubleOk = isequal(std_double, std_double0)
% Compare results of single-precision data (for vs. parfor)
isParforSingleOk = isequal(std_single, par_std_single)
parforSingleAccuracy = std_single / par_std_single
% Compare results of double-precision data (for vs. parfor)
isParforDoubleOk = isequal(std_double, par_std_double)
parforDoubleAccuracy = std_double / par_std_double

Output example :

isForSingleOk =
    1                   % <= true (of course!)
isForDoubleOk =
    1                   % <= true (of course!)
isParforSingleOk =
    0                   % <= false (odd!)
parforSingleAccuracy =
    0.73895227413361    % <= single-precision results are radically different in parfor vs. for
isParforDoubleOk =
    0                   % <= false (odd!)
parforDoubleAccuracy =
    1.00000000000021    % <= double-precision results are almost [but not exactly] the same in parfor vs. for

From my testing, the larger the data array, the bigger the difference is between the results of single-precision data when running in for vs. parfor.
In other words, my experience has been that if you have a huge data matrix, it's better to parallelize it in double-precision if you wish to get [nearly] accurate results. But even so, I find it deeply disconcerting that the results are not exactly identical (at least on R2015a-R2016b on which I tested) even for the native double-precision .
Hmmm... bug?

Upcoming travels - Zürich & Geneva

I will shortly be traveling to clients in Zürich and Geneva, Switzerland. If you are in the area and wish to meet me to discuss how I could bring value to your work with some advanced Matlab consulting or training, then please email me (altmany at gmail):

• Zürich: January 15-17
• Geneva: January 18-21

Happy new year everybody!

The post Quirks with parfor vs. for appeared first on Undocumented Matlab.

1. Using mexCallMATLABWithTrap (documented)
2. Using utGetWarningStatus (undocumented)

Using mexCallMATLABWithTrap (documented)

The first idea is to use documented mexCallMATLABWithTrap function to execute warning(‘query’,…) command using Matlab’s interpreter and then parse the returned result:

bool mxIsWarningEnabled(const char* warningId)
{
    bool enabled = true;
    if (NULL != warningId)
    {
        mxArray *mxCommandResponse = NULL, *mxException = NULL;
        mxArray *args[2];
        /* warning('query', warningId); */
        args[0] = mxCreateString("query");
        args[1] = mxCreateString(warningId);
        mxException = mexCallMATLABWithTrap(1,&mxCommandResponse,2,args,"warning");
        if (NULL == mxException && NULL != mxCommandResponse)
        {
            if (mxIsStruct(mxCommandResponse))
            {
                const mxArray* state_field = mxGetField(mxCommandResponse, 0, "state");
                if (mxIsChar(state_field))
                {
                    char state_value[8] = {0};
                    enabled = (0 == mxGetString(state_field, state_value, 8)) &&
                              (0 == strcmp(state_value,"on"));
                }
            }
            mxDestroyArray(mxCommandResponse);
        }
        else
        {
            /* 'warning' returned with error */
            mxDestroyArray(mxException);
        }
        mxDestroyArray(args[0]);
        mxDestroyArray(args[1]);
    }
    return enabled;
}

This approach is slow, but works fine in most standard situations. See the bottom of this post for a usage example.
However, this approach has an important drawback – we should be careful with recursive calls to the Matlab interpreter (Matlab -> MEX -> Matlab) and with handling Matlab errors in MEX. It is safe only if we use identical standard libraries and compiler to build both MEX and Matlab.
In other cases, for example when MEX is targeted to work with different versions of Matlab, or was built with a different standard library and compiler, etc. – cross boundary handling of errors (which are just C++ exceptions) might lead to unpredictable results, most likely segfaults.

Using utGetWarningStatus (undocumented)

To avoid all the overhead of calling Matlab interpreter and unsafe error handling, we can use some undocumented internal Matlab functions:

/* Link with libut library to pick-up undocumented functions: */
extern "C" void* utGetWarningManagerContext(void);
extern "C" bool  utIsValidMessageIdentifier(const char *warningId);
extern "C" bool  utGetWarningStatus(void* context, const char *warningId);
/*
   Returns true if warning with warningId enabled
   Matlab versions supported/tested: R2008b - R2016b
*/
bool mxIsWarningEnabled(const char *warningId)
{
    bool enabled = true;
    if (NULL != warningId && utIsValidMessageIdentifier(warningId))
    {
        void* context = utGetWarningManagerContext();
        enabled = (NULL != context) && utGetWarningStatus(context, warningId);
    }
    return enabled;
}

Now the code is clean, fast and safe – we bypass the interpreter and work directly with Matlab kernel. All the undocumented functions involved are present in Matlab for at least 10 years and do simple logical checks under the hood.
The standard function mexWarnMsgIdAndTxt uses similar code to check if it should display the warning or just suppress it, and that code remains unchanged since R2008b. This is a good indication of code stability and makes us believe that it will not be changed in future versions of Matlab.
For both workarounds, usage is simple:

if (mxIsWarningEnabled("Matlab:nearlySingularMatrix"))
{
   /* compute rcond */
}
else
{
   /* do something else */
}

The post Checking status of warning messages in MEX appeared first on Undocumented Matlab.

User customizations of Matlab uifigures (click to zoom-in)

A brief introduction to CSS

CSS stands for Cascading Style Sheets. As described on the official webpage of W3C (which governs web standards):

CSS is the language for describing the presentation of Web pages, including colors, layout, and fonts. CSS is independent of HTML. This is referred to as the separation of structure (or: content) from presentation.

CSS rules (or “styles”) can be defined in one of three places:

• A separate file, such as the main.css that Matlab uses for uifigures (this file is found minified in %matlabroot%\toolbox\matlab\uitools\uifigureappjs\release\gbtclient\css)
• An inline block inside the HTML’s <head> section
• Directly within a DOM node

Deciding which of the above to use, is largely a choice of the right tool for the job. Usually, the first two choices should be preferred, as they adhere to the “separation of structure and presentation” idea better. However, in the scope of this demonstration, we’ll be using mostly the 3rd option, because it allows us not to worry about possible CSS precedence issues (suggested read).
The syntax of CSS is generally: selector { property: value }, but it can have other forms as well.

Getting down to business

Let us consider a very basic uifigure that only contains a uitextarea and its label:

classdef DOMdemo < matlab.apps.AppBase
    % Properties that correspond to app components
    properties (Access = public)
        UIFigure      matlab.ui.Figure           % UI Figure
        LabelTextArea matlab.ui.control.Label    % Text Area
        TextArea      matlab.ui.control.TextArea % This is some text.
    end
    methods (Access = private)
        % Code that executes after component creation
        function startupFcn(app)
        end
    end
    % App initialization and construction
    methods (Access = private)
        % Create UIFigure and components
        function createComponents(app)
            % Create UIFigure
            app.UIFigure = uifigure;
            app.UIFigure.Position = [100 100 280 102];
            app.UIFigure.Name = 'UI Figure';
            setAutoResize(app, app.UIFigure, true)
            % Create LabelTextArea
            app.LabelTextArea = uilabel(app.UIFigure);
            app.LabelTextArea.HorizontalAlignment = 'right';
            app.LabelTextArea.Position = [16 73 62 15];
            app.LabelTextArea.Text = 'Text Area';
            % Create TextArea
            app.TextArea = uitextarea(app.UIFigure);
            app.TextArea.Position = [116 14 151 60];
            app.TextArea.Value = {'This is some text.'};
        end
    end
    methods (Access = public)
        % Construct app
        function app = DOMdemo()
            % Create and configure components
            createComponents(app)
            % Register the app with App Designer
            registerApp(app, app.UIFigure)
            % Execute the startup function
            runStartupFcn(app, @startupFcn)
            if nargout == 0
                clear app
            end
        end
        % Code that executes before app deletion
        function delete(app)
            % Delete UIFigure when app is deleted
            delete(app.UIFigure)
        end
    end
end

Let's say we want to modify certain aspects of the TextArea widget, such as the text color, background, and/or horizontal alignment. The workflow for styling elements involves:

1. Find the handle to the webfigure
2. Find the DOM node we want to modify
3. Find the property name that corresponds to the change we want
4. Find a way to manipulate the desired node from Matlab

Step 1: Find the handle to the webfigure

The first thing we need to do is to strategically place a bit of code that would allow us to get the URL of the figure so we can inspect it in our browser:

function startupFcn(app)
   % Customizations (aka "MAGIC GOES HERE"):
   warning off Matlab:HandleGraphics:ObsoletedProperty:JavaFrame
   warning off Matlab:structOnObject
   while true
      try
         win = struct(struct(struct(app).UIFigure).Controller).Container.CEF;
         disp(win.URL);
         break
      catch
         disp('Not ready yet!');
         pause(0.5); % Give the figure (webpage) some more time to load
      end
   end
end

This code waits until the page is sufficiently loaded, and then retrieve its local address (URL). The result will be something like this, which can be directly opened in any browser (outside Matlab):

http://localhost:31415/toolbox/matlab/uitools/uifigureappjs/componentContainer.html?channel=/uicontainer/861ef484-534e-4a50-993e-6d00bdba73a5&snc=88E96E

Step 2: Find the DOM node that corresponds to the component that we want to modify

Loading this URL in an external browser (e.g., Chrome, Firefox or IE/Edge) enables us to use web-development addins (e.g., FireBug) to inspect the page contents (source-code).

Note: in recent Matlab releases, the URL cannot be directly opened in a browser unless we first set Matlab to enable this (thanks to Perttu). We can turn on such browser debugging by running:

matlab.internal.webwindow(' ', 'DebugPort',4040) %note: the space is important!

before any other App, or kill all MatlabWindow processes (from the operating system's task manager) before running the command. Then browse http://localhost:4040/ in your browser - this will display a list of links, one link for each uifigure, based on the uifigure's title (name); click the requested uifigure's link to see and debug the figure's HTML/CSS contents.

Opening the URL inside a browser and inspecting the page contents, we can see its DOM:

Inspecting the DOM in Firefox (click to zoom-in)

createComponents(app):
    app.UIFigure = uifigure;
    app.LabelTextArea = uilabel(app.UIFigure);
    app.TextArea = uitextarea(app.UIFigure);

... and each of them is assigned a random hexadecimal id whenever the app is opened.
Finding the relevant node involved some trial-and-error, but after doing it several times I seem to have found a consistent pattern that can be used to our advantage. Apparently, the nodes with data-tag are always above the element we want to style, sometimes as a direct parent and sometimes farther away. So why do we even need to bother with choosing more accurate nodes than these "tagged" ones? Shouldn't styles applied to the tagged nodes cascade down to the element we care about? Sure, sometimes it works like that, but we want to do better than "sometimes". To that end, we would like to select as relevant a node as possible.
Anyway, the next step in the program is to find the data-tag that corresponds to the selected component. Luckily, there is a direct (undocumented) way to get it:

% Determine the data-tag of the DOM component that we want to modify:
hComponent = app.TextArea;  % handle to the component that we want to modify
data_tag = char(struct(hComponent).Controller.ProxyView.PeerNode.getId);  % this part is generic: can be used with any web-based GUI component

Let's take a look at the elements marked with blue and green borders (in that order) in the DOM screenshot. We see that the data-tag property is exactly one level above these elements, in other words, the first child of the tagged node is an element that contains a widgetid property. This property is very important, as it contains the id of the node that we actually want to change. Think pointers. To summarize this part:

Step 3: Find the CSS property name that corresponds to the change we want

There is no trick here: it's just a matter of going through a list of CSS properties and choosing one that "sounds about right" (there are often several ways to achieve the same visual result with CSS). After we choose the relevant properties, we need to convert them to camelCase as per documentation of dojo.style():

If the CSS style property is hyphenated, the JavaScript property is camelCased. For example: "font-size" becomes "fontSize", and so on.

Note that Matlab R2016a comes bundled with Dojo v1.10.4, rev. f4fef70 (January 11 2015). Other Matlab releases will probably come with other Dojo versions. They will never be the latest version of Dojo, but rather a version that is 1-2 years old. We should keep this in mind when searching the Dojo documentation. We can get the current Dojo version as follows:

>> f=uifigure; drawnow; dojoVersion = matlab.internal.webwindowmanager.instance.windowList(1).executeJS('dojo.version'), delete(f)
dojoVersion =
{"major":1,"minor":10,"patch":4,"flag":"","revision":"f4fef70"}

This tells us that Dojo 1.10.4.f4fef70 is the currently-used version. We can use this information to browse the relevant documentation branch, as well as possibly use different Dojo functions/features.

Step 4: Manipulate the desired element from Matlab

In this demo, we'll use a combination of several commands:

• {matlab.internal.webwindow.}executeJS() – For sending JS commands to the uifigure.
• dojo.query() – for finding nodes inside the DOM.
• dojo.style() (deprecated since v1.8) – for applying styles to the required nodes of the DOM.
  Syntax: dojo.style(node, style, value);
• dojo.setAttr (deprecated since v1.8) – for setting some non-style attributes.
  Syntax: dojo.setAttr(node, name, value);

Consider the following JS commands:

• search the DOM for nodes having a data-tag attribute having the specified value, take their first child of type <div>, and return the value of this child's widgetid attribute:

  ['dojo.getAttr(dojo.query("[data-tag^=''' data_tag '''] > div")[0],"widgetid")']

• search the DOM for nodes with id of widgetid, then take the first element of the result and set its text alignment:

  ['dojo.style(dojo.query("#' widgetId(2:end-1) '")[0],"textAlign","center")']

• append the CSS style defined by {SOME CSS STYLE} to the page (this style can later be used by nodes):

  ['document.head.innerHTML += ''''']);

Putting it all together

It should finally be possible to understand the code that appears in the animated screenshot at the top of this post:

%% 1. Get a handle to the webwindow:
win = struct(struct(struct(app).UIFigure).Controller).Container.CEF;
%% 2. Find which element of the DOM we want to edit (as before):
data_tag = char(struct(app.TextArea).Controller.ProxyView.PeerNode.getId);
%% 3. Manipulate the DOM via a JS command
% ^ always references a class="vc-widget" element.
widgetId = win.executeJS(['dojo.getAttr(dojo.query("[data-tag^=''' data_tag '''] > div")[0],"widgetid")']);
% Change font weight:
dojo_style_prefix = ['dojo.style(dojo.query("#' widgetId(2:end-1) '")[0],'];
win.executeJS([dojo_style_prefix '"fontWeight","900")']);
% Change font color:
win.executeJS([dojo_style_prefix '"color","yellow")']);
% Add an inline css to the HTML :
win.executeJS(['document.head.innerHTML += ''''']);
% Add animation to control:
win.executeJS([dojo_style_prefix '"-webkit-animation","mymove 5s infinite")']);
% Change Dojo theme:
win.executeJS('dojo.setAttr(document.body,''class'',''nihilo'')[0]');
% Center text:
win.executeJS([dojo_style_prefix '"textAlign","center")']);

A similar method for center-aligning the items in a uilistbox is described here (using a CSS text-align directive).
The only thing we need to ensure before running code that manipulates the DOM, is that the page is fully loaded. The easiest way is to include a pause() of several seconds right after the createComponents(app) function (this will not interfere with the creation of the uifigure, as it happens on a different thread). I have been experimenting with another method involving webwindow's PageLoadFinishedCallback callback, but haven’t found anything elegant yet.

A few words of caution

In this demonstration, we invoked Dojo functions via the webwindow's JS interface. For something like this to be possible, there has to exist some form of “bridge” that translates Matlab commands to JS commands issued to the browser and control the DOM. We also know that this bridge has to be bi-directional, because binding Matlab callbacks to uifigure actions (e.g. ButtonPushFcn for uibuttons) is a documented feature.
The extent to which the bridge might allow malicious code to control the Matlab process needs to be investigated. Until then, the ability of webwindows to execute arbitrary JS code should be considered a known vulnerability. For more information, see XSS and related vulnerabilities.

Final remarks

It should be clear now that there are actually lots of possibilities afforded by the new uifigures for user customizations. One would hope that future Matlab releases will expose easier and more robust hooks for CSS/JS customizations of uifigure contents. But until that time arrives (if ever), we can make do with the mechanism shown above.
Readers are welcome to visit the GitHub project dedicated to manipulating uifigures using the methods discussed in this post. Feel free to comment, suggest improvements and ideas, and of course submit some pull requests 🙂
p.s. – it turns out that uifigures can also display MathML. But this is a topic for another post...

The post Customizing uifigures part 2 appeared first on Undocumented Matlab.

>> a = zeros(1e4);
>> tic, b = any(a(:)~=0); toc
Elapsed time is 0.126118 seconds.

Simple enough. In this case, when a is all-zero, the internal search algorithm has no choice but to inspect every element of the array to determine whether it contains any nonzeros. In the more typical case where a contains many nonzeros you would expect the search to terminate almost immediately, as soon as it finds the first nonzero. But that’s not how it works:

>> a = round(rand(1e4));
>> tic, b = any(a(:)~=0); toc
Elapsed time is 0.063404 seconds.

There is significant runtime overhead in constructing the logical array “a(:)~=0”, although the “any(…)” operation apparently terminates at the first true value it finds.
The overhead can be eliminated by taking advantage of the fact that numeric values may be used as logicals in Matlab, with zero implicitly representing false and nonzero representing true. Repeating the above test without “~=0”, we get a huge runtime improvement:

>> a = round(rand(1e4));
>> tic, b = any(a(:)); toc
Elapsed time is 0.000026 seconds.

However, there is no runtime benefit when a is all-zero:

>> a = zeros(1e4);
>> tic, b = any(a(:)); toc
Elapsed time is 0.125120 seconds.

(I do not quite understand this. There should be some runtime benefit from bypassing the logical array construction.)

NaN values

There is also another catch: The above efficiency trick does not work when a contains NaN values (if you consider NaN to be nonzero), e.g.

>> any([0,nan])
ans =
     0

The any function ignores entries that are NaN, meaning it treats NaNs as zero-equivalent. This is inconsistent with the behavior of the inequality operator:

>> any([0,nan]~=0)
ans =
     1

To avoid this problem, an explicit isnan test is needed. Efficiency is not impaired when a contains many nonzeros, but there is a 2x efficiency loss when a is all-zero:

>> a = round(rand(1e4));
>> tic, b = any(a(:)) || any(isnan(a(:))); toc
Elapsed time is 0.000027 seconds.
>> a = zeros(1e4);
>> tic, b = any(a(:)) || any(isnan(a(:))); toc
Elapsed time is 0.256604 seconds.

For testing all-nonzero the NaN problem does not occur:

>> all([1 nan])
ans =
     1

In this context NaN is treated as nonzero and the all-nonzero test is straightforward:

>> a = round(rand(1e4));
>> tic, b = all(a(:)); toc
Elapsed time is 0.000029 seconds.

For testing any-zero and all-zero, use the complements of the above tests:

>> b = ~any(a(:)) || any(isnan(a(:)));  % all zero?
>> b = ~all(a(:));  % any zero?

Efficient find

The find operation can also be optimized by bypassing construction of a logical temporary array, e.g.

>> a = round(rand(1e4));
>> tic, b = find(a(:)~=0, 1); toc
Elapsed time is 0.065697 seconds.
>> tic, b = find(a(:), 1); toc
Elapsed time is 0.000029 seconds.

There is no problem with NaNs in this case; the find function treats NaN as nonzero, e.g.

>> find([0,nan,1], 1)
ans =
     2

The post Zero-testing performance appeared first on Undocumented Matlab.

I’ve been using Matlab for many years now and from time to time I need to profile low-throughput code. When I profile this code sometimes I realize that a computational ‘bottleneck’ is due to a builtin Matlab function (part of the core language). I can often find ways to accelerate such builtin functions and get significant speedup in my code.

I recently found Matlab’s builtin rmfield function being too slow for my needs. It works great when one needs to remove a few fields from a small structure, but in our case we needed to remove thousands of fields from a structure containing about 5000 fields – and this is executed in a function that is called many times inside an external loop. The program was significantly sluggish.

It started when a co-worker asked me to look at a code that looked just slightly more intelligent than this:

for i = 1:5000
    myStruct = rmfield(myStruct,fieldNames{i});
end

Running this code within a tic/toc pair yielded the following results:

>> tic; myFunc(); t1 = toc
t1 =
      25.7713

In my opinion 25.77 secs for such a simple functionality seems like an eternity…

The obvious thing was to change the code to the documented faster (vectorized) version:

>> tic; myStruct = rmfield(myStruct,fieldNames); t2 = toc
t2 =
      0.6097

This is obviously much better but since rmfield is called many times in my application, I needed something even better. So I profiled rmfield and was not happy with the result.

The original code of rmfield (%matlabroot%/toolbox/matlab/datatypes/rmfield.m) looks something like this (I deleted some non-essential code for brevity):

function t = rmfield(s,field)

  % get fieldnames of struct
  f = fieldnames(s);

  % Determine which fieldnames to delete.
  idxremove = [];
  for i=1:length(field)
    j = find(strcmp(field{i},f) == true);
    idxremove = [idxremove;j];
  end

  % set indices of fields to keep
  idxkeep = 1:length(f);
  idxkeep(idxremove) = [];

  % remove the specified fieldnames from the list of fieldnames.
  f(idxremove,:) = [];

  % convert struct to cell array
  c = struct2cell(s);

  % find size of cell array
  sizeofarray = size(c);
  newsizeofarray = sizeofarray;

  % adjust size for fields to be removed
  newsizeofarray(1) = sizeofarray(1) - length(idxremove);

  % rebuild struct
  t = cell2struct(reshape(c(idxkeep,:),newsizeofarray),f);

When I profiled the code, the highlighted row was the bottleneck I was looking for.

First, I noticed the string comparison equals to true part – while '==true' is not the cause of the bottleneck, it does leave an impression of bad coding style 🙁 Perhaps this code was created as some apprentice project, which might also explain its suboptimal performance.

The real performance problem here is that for each field that we wish to remove, rmfield compares it to all existing fields to find its location in a cell array of field names. This is algorithmically inefficient and makes the code hard to understand (just try – it took me hard, long minutes).

So, I created a variant of rmfield.m called fast_rmfield.m, as follows (again, omitting some non-essential code):

function t = fast_rmfield(s,field)
  % get fieldnames of struct
  f = fieldnames(s);
  [f,ia] = setdiff(f,field,'R2012a');

  % convert struct to cell array
  c = squeeze(struct2cell(s));

  % rebuild struct
  t = cell2struct(c(ia,:),f)';

This code is much shorter, easier to explain and maintain, but also (and most importantly) much faster:

>> tic; myStruct = fast_rmfield(myStruct,fieldNames); t3 = toc
t3 =
      0.0302
>> t2/t3
ans =
      20.1893

This resulted in a speedup of ~850x compared to the original version (of 25.77 secs), and ~20x compared to the vectorized version. A nice improvement in my humble opinion…

The point in all this is that we can and should rewrite Matlab builtin functions when they are too slow for our needs, whether it is found to be an algorithmic flaw (as in this case), extraneous sanity checks (as in the case of ismember or datenum), bad default parameters (as in the case of fopen/fwrite or scatter), or merely slow implementation (as in the case of save, cellfun, or the conv family of functions).

A good pattern is to save such code pieces in file names that hint to the original code. In our case, I used fast_rmfield to suggest that it is a faster alternative to rmfield.

Do you know of any other example of a slow implementation in a built-in Matlab function that can be optimized? If so, please leave a comment below.

The post rmfield performance appeared first on Undocumented Matlab.

• PostgreSQL 9.4 installed on the server (Windows 7 PC, but Linux would be equally good)
• DynDNS CNAME pointing at the server (diseserver.mydomain.org)
• CACert.org registration for domain mydomain.org
• Matlab 2015b running on El Capitan

Here are the neccesary steps:

1. First, we need a certificate for the server. We can generate this with OpenSSL:

   $openssl req -out diseserver.csr -new -newkey rsa:2048 -nodes -keyout diseserver.key

   Specify any information you want on the key, but ensure CN=diseserver.mydomain.org.

2. Paste the resulting diseserver.csr file into a new key request at CACert.org. Save the resulting certificate as diseserver.crt on your machine.
3. While still at CACert.org, grab the Class 1 root certificate and save it as root.crt.
4. Put the files diseserver.key, diseserver.crt, and root.crt in the PostgreSQL data directory.
5. Edit your postgresql.conf file:

   ssl = on
   ssl_cert_file = 'diseserver.crt'  # (change requires restart)
   ssl_key_file  = 'diseserver.key'  # (change requires restart)
   ssl_ca_file   = 'root.crt'        # (change requires restart)
   

6. Restart the PostgreSQL server. The server will now permit SSL connections, a necessary pre-condition for certificate authentication.
7. We now add 2 lines to pg_hba.conf:

   hostnossl  all    all   0.0.0.0/0   reject
   hostssl	 mytable  all   0.0.0.0/0   cert map=ssl clientcert=1
   

   The first line causes all non-SSL connections to be rejected. The second allows certificate logins for mytable using the map ssl that is defined in pg_ident.conf:

   ssl  /^(.*).mydomain\.org$ \1

   this line extracts the username prefix from CN=username.mydomain.org.

8. Now we need to generate client certificates. PostgreSQL expects these to be in ~/.postgresql (Windows %appdata%\postgresql\):

   $mkdir ~/.postgresql
   $cd ~/.postgresql
   $openssl req -out postgresql.csr -new -newkey rsa:2048 -nodes -keyout postgresql.key
   

   for this key, make CN=username.mydomain.org.

9. Again, paste the resulting postgresql.csr file into CACert.org, saving the certificate as postgresql.crt.
10. Test this:

    $psql "sslmode=verify-full host=diseserver.mydomain.org dbname=effect user=username"

    The server should respond:

    psql (9.4.6, server 9.4.1)
    SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES256-GCM-SHA384, bits: 256, compression: off)
    

11. Next we need to convert our key into pkcs8 format so that Java can read it:

    $openssl pkcs8 -topk8 -inform PEM -outform DER -in postgresql.key -out postgresql.pk8 -nocrypt

12. Next, ensure that we have the correct version of the JDBC driver (Java-to-database connector). From the Mac command line:

    $java -version
    java version "1.8.0_05"
    

    and in Matlab:

    >> version -java
    ans =
    Java 1.7.0_75-b13 with Oracle Corporation Java HotSpot(TM) 64-Bit Server VM mixed mode
    

    This shows that although we have Java 8 installed on El Capitan (at the OS level), Matlab uses a private Java 7 version. So we need the correct version of the jdbc on our static java classpath that is used by Matlab:

    ~/Matlab/postgresql-9.4.1208.jre7.jar

13. The next part is very poorly documented in both the MathWorks and the PostgreSQL documentation, but I found it in Russel Gray’s Basildon Coder blog: We need to use the jdbc postgresql driver to check the client certificate. To do this, we need a custom SSLSocketFactory – LibPQFactory. This will grab our certificate and key from ~/.postgresql and present them to the server. The url is (note the trailing &):

    jdbc:postgresql://diseserver.mydomain.org/mytable?ssl=true&sslfactory=org.postgresql.ssl.jdbc4.LibPQFactory&sslmode=verify-full&

14. Next we need the username. Rather than hard-coding this in the source code, we get the system username:

    >> username = java.lang.System.getProperty('user.name');

15. Bundle this all up in a Matlab function, stripping the trailing CR from the username:

    function dbtest
       driver = 'org.postgresql.Driver';
       [~,username] = system('whoami');
       url = 'jdbc:postgresql://diseserver.mydomain.org/mytable?ssl=true&sslfactory=org.postgresql.ssl.jdbc4.LibPQFactory&sslmode=verify-full&';
       myconn = database('mytable', username, '', driver, url);
       if ~isempty(myconn.Message)
          fprintf(2,'%s\n', myconn.Message);
       else
          fprintf(1, 'Connected!\n');
       end
    end
    

Now we can connect from the Matlab command line or a Matlab program.
What if we’re deployed? We also need to add the contents of our .postgresql directory, plus the jdbc jar file to our deployed app:

>> mcc -m dbtest.m -a ~/.postgresql -a ~/Matlab/postgresql-9.4.1208.jre7.jar

Let’s test the compiled program from the OS command line:

$./run_dbtest.sh /Applications/Matlab/Matlab_Runtime/v90
Connected!

Note that the key and certificates are part of the encrypted bundle produced by Matlab’s mcc compiler.
I hope this helps someone!
Yair’s note: the Matlab code above uses Matlab’s Database Toolbox (specifically, the database function) to connect to the database. In future posts I plan to show how we can connect Matlab directly to a database via JDBC. This topic is covered in detail in chapter 2 of my Matlab-Java programming secrets book.
p.s. – this blog celebrates a 7-year anniversary tomorrow: I published my very first post here on March 19, 2009, showing how to change Matlab’s command-window colors (a post that later led to the now-famous cprintf utility). It’s been a long and very interesting ride indeed, but I have no plans to retire anytime soon 🙂

The post Secure SSL connection between Matlab and PostgreSQL appeared first on Undocumented Matlab.