<?php

function my_replace ($str) {
    $expr = '~
        (
            # opening bracket or paren
            (?:\(|\[) 
                 # optional opening tag
                 (?:<([a-z])>)?
                 # either up to 3 digits or up to 3 alphas
                 (?:[a-z]{1,3}|[0-9]{1,3})
                 # optional closing tag
                 (?:</\2>)?
                 # optional dot
                 \.?
             # closing bracket or paren
             (?:\)|])
         )
         # optional whitespace
         \s*
         # grab the rest of the string
         (.+)
    ~ix';
    return preg_replace($expr, '%%$1|$3', $str);
}

$strings = array("(1)blahblah", "(<i>iv</i>.) blahblah", "[b] &nbsp;some stuff", "[123) <-- mismatch");
	
foreach ($strings as $str) {
	echo my_replace($str)."\n";
}

%%(1)|blahblah
%%(<i>iv</i>.)|blahblah
%%[b]|&nbsp;some stuff
%%[123)|<-- mismatch