Symbolic Toolbox

clc;
close all;

syms x1 x2 x3
a='x1^2+2*x2^2-3*x3^2-6*x1*x2+8*x1*x3-4*x2*x3'

A=[diff(diff(a,x1),x1),diff(diff(a,x1),x2),diff(diff(a,x1),x3);...
    diff(diff(a,x2),x1),diff(diff(a,x2),x2),diff(diff(a,x2),x3);...
    diff(diff(a,x3),x1),diff(diff(a,x3),x2),diff(diff(a,x3),x3)]

delta=[];
for i=1:3
    disp '  ----';
    i
    A(1:i,1:i)
    delta=det(A(1:i,1:i))
end