Here comes the final version for now. To get it into a full function would take more time. This is enough for my needs.
load testfile.mat
%% test 2 with adding columns for color and marker
%% add color column
%% 1 2 3 4 5 6
%% white yellow orgnge green blue red
%% 'y' [1 0.6 0] 'g' 'b' 'r'
colorNew = rowfun(@(clr){ChangeColor(clr)}, testTable,"InputVariables","color","OutputVariableNames",'colorNew');
%% add marker column
%% AX point '.'
%% AY square 's'
%% AZ upwards triangle '^'
%% BX circle 'o'
%% BY diamond 'd'
%% BZ downward triangle 'v'
markerNew = rowfun(@(marker){ChangeMarker(marker)}, testTable,'InputVariables','marker','OutputVariableNames', 'markerNew');
%% add size to each marker
%% '.' 50, all other 20
sizeNew = rowfun(@(marker){ChangeSize(marker)}, testTable,'InputVariables','marker','OutputVariableNames', 'sizeNew');
% add size,color, marker to table
testTable= [testTable,sizeNew,colorNew,markerNew];
% convert to string
testTable.color = string(testTable.color);
testTable.marker = string(testTable.marker);
% create a list of all entries for the legend.
strFullLegend = rowfun(@(in1,in2){two2oneString(in1,in2)},testTable,'InputVariables',{'marker' 'color'},'OutputVariableNames','str');
strFullLegend = table2cell(strFullLegend); %legend needs to be a cell array
% group the table entries
[G,i1,i2] = findgroups(testTable.color,testTable.marker);
GroupTable = table(string(i2), string(i1));
strGroupLegend= rowfun(@(in1,in2){two2oneString(in1,in2)},GroupTable,'InputVariables',{'Var1' 'Var2'},'OutputVariableNames','str');
strGroupLegend = table2cell(strGroupLegend); %legend needs to be a cell array
figure %% with full legend
hold on
t=rowfun(@(tx,ty,tsz,tclr,tmrk){scatter(tx,ty,tsz{:},tclr{:},tmrk{:})}, testTable,"InputVariables",{'x-axis' 'y-axis' 'sizeNew' 'colorNew' 'markerNew'});
hold off
xlabel("Frequency [Hz]")
ylabel(["Magnetic field"; "strength [dB\muA/m]"])
legend(strFullLegend{:},'Location','northeastoutside')

% replace all double entries by ''
% preallocate cell array with {''}
strShortLegend = repmat({''},size(strFullLegend));
for i= 1:max(G)
idx = G==i; %creat index list for all groups
if sum(idx)>1
%find first one and replace with strGroupLegend value
flag = 1;
for f = 1:length(idx)
if idx(f)==1 && flag ==1
flag = 0;
idx(f) = 1;
else
idx(f) = 0;
end
end
else
% do nothing
end
%than write in the rest ''
strShortLegend(idx)= strGroupLegend(i,1);
end
figure %% with short legend
hold on
t=rowfun(@(tx,ty,tsz,tclr,tmrk){scatter(tx,ty,tsz{:},tclr{:},tmrk{:})}, testTable,"InputVariables",{'x-axis' 'y-axis' 'sizeNew' 'colorNew' 'markerNew'});
hold off
xlabel("Frequency [Hz]")
ylabel(["Magnetic field"; "strength [dB\muA/m]"])
legend(strShortLegend{:},'Location','northeastoutside')

function clr = ChangeColor(clr_in)
switch clr_in
case 3
clr = [1 0.6 0];
case 4
clr = [0 1 0];
case 5
clr = [0 0 1];
case 6
clr = [1 0 0];
end
end
function mkr = ChangeMarker(mkr_in)
switch mkr_in
case 'A-X'
mkr = '.';
sz = 60;
case 'A-Y'
mkr = 's';
sz = 30;
case 'A-Z'
mkr = '^';
sz = 30;
case 'B-X'
mkr = 'o';
sz = 30;
case 'B-Y'
mkr = 'd';
sz = 30;
case 'B-Z'
mkr = 'v';
sz = 30;
end
end
function sz = ChangeSize(mkr_in)
switch mkr_in
case 'A-X'
sz = 50;
case 'A-Y'
sz = 20;
case 'A-Z'
sz = 20;
case 'B-X'
sz = 20;
case 'B-Y'
sz = 20;
case 'B-Z'
sz = 20;
end
end
function str = two2oneString(in1,in2)
str = sprintf('%s,%s',in1,in2);
end

